mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 18:37:24 +00:00
perf(orchestration): stop coordinator check and worker-list from rescanning Run history
- check: the empty-check hint projects only workers that can still be running or owe an action - worker-list: later pages reuse the first page's inventory counts from the cursor - check: read completion reports for follow-ups in one query - outstanding-delivery lookups and the worker seat-grab check use their indexes - hot by-id lookups name their columns so prepared statements are cached - stop-archive checks archive existence without reading the archive
This commit is contained in:
@@ -15,6 +15,7 @@ import {
|
||||
import { dispatchAssigneeOrcaSessionId } from '../../addressing/session-lineage/dispatch-assignee-orca-session-id'
|
||||
import { assertTakesNewWork } from '../worker-terminal/worker-terminal-transfer'
|
||||
import { retireRemotePlacement } from '../remote-owned/remote-placement-hold'
|
||||
import { DISPATCH_CONTEXT_COLUMN_LIST } from '../row-column-lists'
|
||||
|
||||
export function createDispatchContext(
|
||||
this: OrchestrationDb,
|
||||
@@ -126,9 +127,9 @@ export function getDispatchContextById(
|
||||
this: OrchestrationDb,
|
||||
dispatchId: string
|
||||
): DispatchContextRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(dispatchId) as
|
||||
| DispatchContextRow
|
||||
| undefined
|
||||
return this.db
|
||||
.prepare(`SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts WHERE id = ?`)
|
||||
.get(dispatchId) as DispatchContextRow | undefined
|
||||
}
|
||||
|
||||
export function commitDispatchLaunchTokenHash(
|
||||
|
||||
@@ -8,10 +8,11 @@ import { ORCHESTRATION_LEGACY_RUN_ID as LEGACY_RUN_ID } from '../../../../../sha
|
||||
import { LEGACY_CONTRACT_VERSION } from '../contract-constants'
|
||||
import { generateId } from '../generated-id'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { LEGACY_ADOPTION_COLUMN_LIST } from '../row-column-lists'
|
||||
|
||||
export function getLegacyAdoption(this: OrchestrationDb): LegacyAdoptionRow | undefined {
|
||||
return this.db
|
||||
.prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?')
|
||||
.prepare(`SELECT ${LEGACY_ADOPTION_COLUMN_LIST} FROM legacy_adoptions WHERE source_run_id = ?`)
|
||||
.get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { MessageType, MessageRow } from '../orchestration-records'
|
||||
import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
|
||||
import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { MESSAGE_COLUMN_LIST } from '../row-column-lists'
|
||||
|
||||
const MESSAGE_ID_UPDATE_BATCH_SIZE = 500
|
||||
const MESSAGE_MUTATION_SAVEPOINT = 'message_id_mutation'
|
||||
@@ -151,9 +152,9 @@ export function getAllMessages(this: OrchestrationDb, toHandle: string, limit =
|
||||
}
|
||||
|
||||
export function getMessageById(this: OrchestrationDb, id: string): MessageRow | undefined {
|
||||
const message = this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as
|
||||
| MessageRow
|
||||
| undefined
|
||||
const message = this.db
|
||||
.prepare(`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id = ?`)
|
||||
.get(id) as MessageRow | undefined
|
||||
return message ? exposeMessageTimestamps(message) : undefined
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner'
|
||||
import { notifyPeerMailQueued } from '../../federation/peer-mail/peer-mail-waiters'
|
||||
import { ORCHESTRATION_UNBOUND_RUN_ID as UNBOUND_RUN_ID } from '../../../../../shared/orchestration/orchestration-rpc-contract'
|
||||
import { MESSAGE_COLUMN_LIST } from '../row-column-lists'
|
||||
|
||||
const MESSAGE_INSERT_SAVEPOINT = 'message_insert_batch'
|
||||
const WORKER_DONE_MESSAGE_SAVEPOINT = 'worker_done_message_commit'
|
||||
@@ -86,7 +87,9 @@ export function insertMessage(this: OrchestrationDb, msg: MessageInsert): Messag
|
||||
// Mail filed in a peer Run is queued home by trigger in this statement.
|
||||
notifyPeerMailQueued(this.db)
|
||||
return exposeMessageTimestamps(
|
||||
this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow
|
||||
this.db
|
||||
.prepare(`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id = ?`)
|
||||
.get(id) as MessageRow
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import type { MessageRow } from '../orchestration-records'
|
||||
import { MESSAGE_COLUMN_LIST } from '../row-column-lists'
|
||||
import { exposeMessageListTimestamps } from '../utc-timestamp'
|
||||
|
||||
/** One read for a batch of ids, keyed by id; unknown ids are absent. */
|
||||
export function getMessagesByIds(
|
||||
db: OrchestrationDb,
|
||||
ids: readonly string[]
|
||||
): Map<string, MessageRow> {
|
||||
if (ids.length === 0) {
|
||||
return new Map()
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the column list names every MessageRow column.
|
||||
const rows = db.db
|
||||
.prepare(
|
||||
`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id IN (SELECT value FROM json_each(?))`
|
||||
)
|
||||
.all(JSON.stringify(ids)) as MessageRow[]
|
||||
return new Map(exposeMessageListTimestamps(rows).map((row) => [row.id, row]))
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from '../orchestration-db'
|
||||
import { plansOf } from '../query-plan-capture.test-support'
|
||||
|
||||
describe('outstanding mailbox Delivery lookups', () => {
|
||||
let db: OrchestrationDb
|
||||
|
||||
beforeEach(() => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
db.close()
|
||||
})
|
||||
|
||||
it('read the outstanding index instead of every Delivery ever taken', () => {
|
||||
const plans = plansOf(db, ['deliveries'], () => {
|
||||
db.hasOutstandingMailboxDelivery('run:run_a')
|
||||
db.getOutstandingMailboxDelivery('run:run_a')
|
||||
db.fenceUnacknowledgedMailboxDeliveries('run:run_a')
|
||||
})
|
||||
|
||||
expect(plans.filter((detail) => detail.startsWith('SCAN deliveries'))).toEqual([])
|
||||
expect(
|
||||
plans.filter((detail) => detail.includes('idx_deliveries_one_outstanding'))
|
||||
).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
@@ -5,9 +5,15 @@ import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { exposeDeliveryTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
|
||||
import { requireMailboxConsumer } from './mailbox-consumer'
|
||||
import { ORCHESTRATION_DELIVERY_BATCH_LIMIT } from './mailbox-routing-page'
|
||||
import { DELIVERY_COLUMN_LIST } from '../row-column-lists'
|
||||
|
||||
// Repeats the outstanding index's predicate so lookups use it; '' marks pre-mailbox rows, never a mailbox.
|
||||
const OUTSTANDING_IN_MAILBOX = "mailbox_handle = ? AND mailbox_handle != ''"
|
||||
|
||||
export function getDeliveryRaw(this: OrchestrationDb, id: string): DeliveryRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id) as DeliveryRow | undefined
|
||||
return this.db.prepare(`SELECT ${DELIVERY_COLUMN_LIST} FROM deliveries WHERE id = ?`).get(id) as
|
||||
| DeliveryRow
|
||||
| undefined
|
||||
}
|
||||
|
||||
export function getDeliveryMessages(this: OrchestrationDb, delivery: DeliveryRow): MessageRow[] {
|
||||
@@ -88,7 +94,7 @@ function selectMailboxDelivery(
|
||||
ORCHESTRATION_DELIVERY_BATCH_LIMIT
|
||||
)
|
||||
const outstanding = db.db
|
||||
.prepare('SELECT id FROM outstanding_deliveries WHERE mailbox_handle = ?')
|
||||
.prepare(`SELECT id FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX}`)
|
||||
.get(params.mailboxHandle)
|
||||
if (typeof outstanding?.id === 'string') {
|
||||
const existing = requireDeliveryRaw(db, outstanding.id)
|
||||
@@ -233,7 +239,7 @@ export function hasOutstandingMailboxDelivery(
|
||||
): boolean {
|
||||
return Boolean(
|
||||
this.db
|
||||
.prepare('SELECT 1 FROM outstanding_deliveries WHERE mailbox_handle = ? LIMIT 1')
|
||||
.prepare(`SELECT 1 FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX} LIMIT 1`)
|
||||
.get(mailboxHandle)
|
||||
)
|
||||
}
|
||||
@@ -243,7 +249,9 @@ export function getOutstandingMailboxDelivery(
|
||||
mailboxHandle: string
|
||||
): { id: string; messageIds: string[] } | undefined {
|
||||
const row = this.db
|
||||
.prepare('SELECT id, message_ids FROM outstanding_deliveries WHERE mailbox_handle = ? LIMIT 1')
|
||||
.prepare(
|
||||
`SELECT id, message_ids FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX} LIMIT 1`
|
||||
)
|
||||
.get(mailboxHandle)
|
||||
if (typeof row?.id !== 'string' || typeof row.message_ids !== 'string') {
|
||||
return undefined
|
||||
@@ -260,7 +268,7 @@ export function fenceUnacknowledgedMailboxDeliveries(
|
||||
): void {
|
||||
this.db
|
||||
.prepare(
|
||||
"UPDATE deliveries SET status = 'fenced' WHERE mailbox_handle = ? AND status = 'outstanding'"
|
||||
`UPDATE deliveries SET status = 'fenced' WHERE ${OUTSTANDING_IN_MAILBOX} AND status = 'outstanding'`
|
||||
)
|
||||
.run(mailboxHandle)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { MutationReceiptRow } from '../orchestration-records'
|
||||
import { OrchestrationError } from '../../orchestration-error'
|
||||
import { ensureMutationReceiptCapacity } from './mutation-receipt-capacity'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { MUTATION_RECEIPT_COLUMN_LIST } from '../row-column-lists'
|
||||
|
||||
export function getOrCreateLocalMutationCallerFingerprint(this: OrchestrationDb): string {
|
||||
if (this.localMutationCallerFingerprint) {
|
||||
@@ -190,7 +191,7 @@ export function getMutationReceipt(
|
||||
): MutationReceiptRow | undefined {
|
||||
return this.db
|
||||
.prepare(
|
||||
`SELECT * FROM mutation_receipts
|
||||
`SELECT ${MUTATION_RECEIPT_COLUMN_LIST} FROM mutation_receipts
|
||||
WHERE caller_fingerprint = ? AND request_id = ?`
|
||||
)
|
||||
.get(callerFingerprint, requestId) as MutationReceiptRow | undefined
|
||||
|
||||
@@ -4,3 +4,9 @@ import { ORCA_SESSION_ADDRESS_PREFIX } from '../../../../shared/orca-session-add
|
||||
export function orcaSessionAddressSql(orcaSessionIdSql: string): string {
|
||||
return `('${ORCA_SESSION_ADDRESS_PREFIX}' || ${orcaSessionIdSql})`
|
||||
}
|
||||
|
||||
/** The bare id a session address names, NULL for anything else; comparing ids lets the id index serve. */
|
||||
export function orcaSessionIdOfAddressSql(addressSql: string): string {
|
||||
return `(CASE WHEN substr(${addressSql}, 1, ${ORCA_SESSION_ADDRESS_PREFIX.length}) = '${ORCA_SESSION_ADDRESS_PREFIX}'
|
||||
THEN substr(${addressSql}, ${ORCA_SESSION_ADDRESS_PREFIX.length + 1}) END)`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { expect, vi } from 'vitest'
|
||||
import type { OrchestrationDb } from './orchestration-db'
|
||||
|
||||
/** The query plan of every statement `act` prepares that reads or writes one of `tables`. */
|
||||
export function plansOf(db: OrchestrationDb, tables: string[], act: () => void): string[] {
|
||||
const prepare = vi.spyOn(db.db, 'prepare')
|
||||
act()
|
||||
const statements = prepare.mock.calls
|
||||
.map(([sql]) => sql)
|
||||
.filter((sql) => tables.some((table) => sql.includes(table)))
|
||||
prepare.mockRestore()
|
||||
expect(statements.length).toBeGreaterThan(0)
|
||||
return statements.flatMap((sql) =>
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: EXPLAIN QUERY PLAN rows carry a detail column.
|
||||
(db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]).map(
|
||||
({ detail }) => detail
|
||||
)
|
||||
)
|
||||
}
|
||||
@@ -1,25 +1,9 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from '../orchestration-db'
|
||||
import { plansOf } from '../query-plan-capture.test-support'
|
||||
import { countRemoteDispatches, listRemoteDispatches } from './remote-dispatch-records'
|
||||
import { closeRemoteRunLink, openRemoteRunLink } from './remote-run-links'
|
||||
|
||||
/** The query plan of every statement `act` prepares that reads or writes one of `tables`. */
|
||||
function plansOf(db: OrchestrationDb, tables: string[], act: () => void): string[] {
|
||||
const prepare = vi.spyOn(db.db, 'prepare')
|
||||
act()
|
||||
const statements = prepare.mock.calls
|
||||
.map(([sql]) => sql)
|
||||
.filter((sql) => tables.some((table) => sql.includes(table)))
|
||||
prepare.mockRestore()
|
||||
expect(statements.length).toBeGreaterThan(0)
|
||||
return statements.flatMap((sql) =>
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: EXPLAIN QUERY PLAN rows carry a detail column.
|
||||
(db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]).map(
|
||||
({ detail }) => detail
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
describe('remote-owned queries read through their indexes', () => {
|
||||
let db: OrchestrationDb
|
||||
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import type { AttemptObservationStorageRow } from './attempt-observation-store'
|
||||
import type {
|
||||
DeliveryRow,
|
||||
DispatchContextRow,
|
||||
FederatedDispatchRow,
|
||||
LegacyAdoptionRow,
|
||||
MessageRow,
|
||||
MutationReceiptRow,
|
||||
RemoteDispatchAttachmentRow,
|
||||
RunRow,
|
||||
TaskRow
|
||||
@@ -147,3 +152,79 @@ export const ATTEMPT_OBSERVATION_FACT_COLUMN_LIST = selectColumns(ATTEMPT_OBSERV
|
||||
export const REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST = selectColumns(
|
||||
REMOTE_DISPATCH_ATTACHMENT_COLUMNS
|
||||
)
|
||||
|
||||
/** A projection naming every `Row` column; leaving one out fails compilation. */
|
||||
function everyColumnOf<Row>() {
|
||||
return <const Columns extends readonly (keyof Row & string)[]>(
|
||||
columns: Columns & (Exclude<keyof Row, Columns[number]> extends never ? unknown : never)
|
||||
): string => selectColumns(columns)
|
||||
}
|
||||
|
||||
export const MESSAGE_COLUMN_LIST = everyColumnOf<MessageRow>()([
|
||||
'id',
|
||||
'run_id',
|
||||
'delivery_contract',
|
||||
'from_handle',
|
||||
'to_handle',
|
||||
'subject',
|
||||
'body',
|
||||
'type',
|
||||
'priority',
|
||||
'thread_id',
|
||||
'payload',
|
||||
'read',
|
||||
'sequence',
|
||||
'created_at',
|
||||
'delivered_at',
|
||||
'sender_pane_key',
|
||||
'answer_sender_snapshot',
|
||||
'pointer_enter_pending',
|
||||
'pointer_pty_id',
|
||||
'pointer_process_incarnation'
|
||||
])
|
||||
|
||||
export const DELIVERY_COLUMN_LIST = everyColumnOf<DeliveryRow>()([
|
||||
'id',
|
||||
'run_id',
|
||||
'mailbox_handle',
|
||||
'consumer_generation',
|
||||
'message_ids',
|
||||
'status',
|
||||
'created_at',
|
||||
'acknowledged_at'
|
||||
])
|
||||
|
||||
export const MUTATION_RECEIPT_COLUMN_LIST = everyColumnOf<MutationReceiptRow>()([
|
||||
'caller_fingerprint',
|
||||
'request_id',
|
||||
'method',
|
||||
'payload_hash',
|
||||
'state',
|
||||
'receipt',
|
||||
'created_at',
|
||||
'updated_at'
|
||||
])
|
||||
|
||||
export const LEGACY_ADOPTION_COLUMN_LIST = everyColumnOf<LegacyAdoptionRow>()([
|
||||
'source_run_id',
|
||||
'adopted_run_id',
|
||||
'scheduler_state_lost',
|
||||
'adopted_at'
|
||||
])
|
||||
|
||||
export const FEDERATED_DISPATCH_COLUMN_LIST = everyColumnOf<FederatedDispatchRow>()([
|
||||
'dispatch_id',
|
||||
'environment_id',
|
||||
'environment_name',
|
||||
'peer_fingerprint',
|
||||
'remote_runtime_epoch',
|
||||
'protocol_version',
|
||||
'remote_worktree_id',
|
||||
'remote_terminal_handle',
|
||||
'to_home_imported_sequence',
|
||||
'to_home_acknowledged_sequence',
|
||||
'home_auto_release_at',
|
||||
'home_auto_release_state',
|
||||
'created_at',
|
||||
'updated_at'
|
||||
])
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type Database from '../../../../sqlite/sync-database'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { currentRunCoordinatorSessionAddressSql } from './run-coordinator-orca-session'
|
||||
import { orcaSessionAddressSql } from '../orca-session-address-sql'
|
||||
import { orcaSessionAddressSql, orcaSessionIdOfAddressSql } from '../orca-session-address-sql'
|
||||
|
||||
/** Mail to an active Dispatch assignee's address in the same Run is that worker's, not coordinator mail. */
|
||||
export function activeDispatchOwnsAddressSql(runIdSql: string, addressSql: string): string {
|
||||
@@ -22,16 +22,20 @@ export function activeDispatchOwnsAddressSql(runIdSql: string, addressSql: strin
|
||||
* because first_bound_at has no milliseconds, so a bind in the second a Dispatch settled is not a grab.
|
||||
*/
|
||||
export function workerSeatGrabSql(aliasRow: string): string {
|
||||
return `EXISTS (
|
||||
// One EXISTS per address form reads its assignee index; unary + keeps the stat-less planner off the Run index it ties with.
|
||||
const grabbedAs = (assigneeMatchSql: string) => `EXISTS (
|
||||
SELECT 1 FROM dispatch_contexts AS grab
|
||||
WHERE grab.run_id = ${aliasRow}.run_id
|
||||
AND (grab.assignee_handle = ${aliasRow}.terminal_handle
|
||||
OR ${orcaSessionAddressSql('grab.assignee_orca_session_id')} = ${aliasRow}.terminal_handle)
|
||||
WHERE ${assigneeMatchSql}
|
||||
AND +grab.run_id = ${aliasRow}.run_id
|
||||
AND NOT (grab.creator_handle IS NOT NULL AND grab.creator_handle = grab.assignee_handle)
|
||||
AND julianday(grab.created_at) <= julianday(${aliasRow}.first_bound_at)
|
||||
AND (grab.status IN ('pending', 'dispatched')
|
||||
OR julianday(datetime(grab.completed_at)) > julianday(${aliasRow}.first_bound_at))
|
||||
)`
|
||||
return `(${grabbedAs(`grab.assignee_handle = ${aliasRow}.terminal_handle`)}
|
||||
OR ${grabbedAs(
|
||||
`grab.assignee_orca_session_id = ${orcaSessionIdOfAddressSql(`${aliasRow}.terminal_handle`)}`
|
||||
)})`
|
||||
}
|
||||
|
||||
/** Whether `addressSql` is a coordinator address of the Run `runIdSql`, seat grabs excluded. */
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from '../orchestration-db'
|
||||
import { plansOf } from '../query-plan-capture.test-support'
|
||||
|
||||
describe('worker seat-grab check', () => {
|
||||
let db: OrchestrationDb
|
||||
|
||||
beforeEach(() => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
db.close()
|
||||
})
|
||||
|
||||
it("reads the address's own Dispatches, not every Dispatch in the Run", () => {
|
||||
const plans = plansOf(db, ['dispatch_contexts'], () =>
|
||||
db.getRunMailboxOwnerIdsForHandle('term_coordinator')
|
||||
)
|
||||
|
||||
expect(plans.filter((detail) => detail.includes('idx_dispatch_run_status'))).toEqual([])
|
||||
expect(plans).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.stringContaining('idx_dispatch_assignee_handle'),
|
||||
expect.stringContaining('idx_dispatch_assignee_orca_session_id')
|
||||
])
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -17,7 +17,8 @@ export const DERIVED_DELIVERY_SCHEMA_SQL = `
|
||||
CREATE TRIGGER IF NOT EXISTS trg_deliveries_one_outstanding
|
||||
AFTER INSERT ON deliveries
|
||||
WHEN NEW.mailbox_handle != '' AND EXISTS (
|
||||
SELECT 1 FROM outstanding_deliveries WHERE mailbox_handle = NEW.mailbox_handle LIMIT 1 OFFSET 1
|
||||
SELECT 1 FROM outstanding_deliveries
|
||||
WHERE mailbox_handle = NEW.mailbox_handle AND mailbox_handle != '' LIMIT 1 OFFSET 1
|
||||
)
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'Mailbox already has an outstanding delivery');
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { fleetWorkerIdentity } from '../../../../../shared/orchestration/fleet/orchestration-fleet-agent-status-evidence'
|
||||
import { SETTLED_WORKER_STATES } from '../../../../../shared/orchestration/fleet/orchestration-fleet-worker-projection'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { WORKER_TERMINAL_HANDED_ON_SQL } from './worker-terminal-inventory-counts'
|
||||
|
||||
const SETTLED_WORKER_STATE_LIST = [...SETTLED_WORKER_STATES].map((state) => `'${state}'`).join(', ')
|
||||
|
||||
type CandidateRow = {
|
||||
dispatch_id: string
|
||||
agent_terminal_handle: string | null
|
||||
pane_key: string | null
|
||||
may_run: number
|
||||
may_owe_action: number
|
||||
}
|
||||
|
||||
/** The keys worker-list's status index counts owners by, from the same identity. */
|
||||
function ownerKeys(row: CandidateRow): string[] {
|
||||
const identity = fleetWorkerIdentity({
|
||||
agentTerminalHandle: row.agent_terminal_handle,
|
||||
paneKey: row.pane_key
|
||||
})
|
||||
if (identity.kind === 'unidentifiable') {
|
||||
return []
|
||||
}
|
||||
const handleKey = `handle:${identity.terminalHandle}`
|
||||
return identity.kind === 'pane_and_terminal'
|
||||
? [handleKey, `pane:${identity.paneKey}`]
|
||||
: [handleKey]
|
||||
}
|
||||
|
||||
/**
|
||||
* The Run's Dispatches whose worker-list outcome can be in_progress or whose next action can be
|
||||
* other than none, plus one other owner of each handle or pane they use, since a shared identity
|
||||
* withholds status evidence. Everything else is settled history that projects to none.
|
||||
*/
|
||||
export function listWorkerReportWaitCandidates(this: OrchestrationDb, runId: string): string[] {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the query selects exactly these columns.
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT d.id AS dispatch_id,
|
||||
CASE WHEN ${WORKER_TERMINAL_HANDED_ON_SQL} THEN NULL
|
||||
ELSE COALESCE(w.agent_terminal_handle, d.assignee_handle) END AS agent_terminal_handle,
|
||||
CASE WHEN ${WORKER_TERMINAL_HANDED_ON_SQL} THEN NULL
|
||||
ELSE COALESCE(r.pane_key, d.assignee_pane_key) END AS pane_key,
|
||||
(d.status IN ('pending', 'dispatched')
|
||||
OR EXISTS (SELECT 1 FROM attempt_observation_facts f
|
||||
WHERE f.dispatch_id = d.id AND f.facet = 'liveness')) AS may_run,
|
||||
((COALESCE(w.state, 'unsupervised') NOT IN (${SETTLED_WORKER_STATE_LIST})
|
||||
AND w.stage IS NOT 'released' AND r.release_state IS NOT 'released'
|
||||
AND NOT (d.status = 'completed' AND w.dispatch_id IS NULL AND r.id IS NULL))
|
||||
OR (r.ownership_state = 'owned' AND r.release_state IS NOT 'released'
|
||||
AND r.release_state IS NOT 'retained')
|
||||
) AS may_owe_action
|
||||
FROM dispatch_contexts d
|
||||
LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id
|
||||
LEFT JOIN worker_terminal_resources r ON r.owner_dispatch_id = d.id
|
||||
WHERE d.run_id = ?`
|
||||
)
|
||||
.all(runId) as CandidateRow[]
|
||||
// Without a possibly running worker the hint cannot apply, so nothing needs projecting.
|
||||
if (!rows.some((row) => row.may_run === 1)) {
|
||||
return []
|
||||
}
|
||||
const isCandidate = (row: CandidateRow) => row.may_run === 1 || row.may_owe_action === 1
|
||||
const kept = rows.filter(isCandidate)
|
||||
// One more owner already makes a key shared; further owners change nothing.
|
||||
const unshared = new Set(kept.flatMap(ownerKeys))
|
||||
for (const row of rows) {
|
||||
const claimed = isCandidate(row) ? [] : ownerKeys(row).filter((key) => unshared.has(key))
|
||||
if (claimed.length > 0) {
|
||||
claimed.forEach((key) => unshared.delete(key))
|
||||
kept.push(row)
|
||||
}
|
||||
}
|
||||
return [...new Set(kept.map((row) => row.dispatch_id))]
|
||||
}
|
||||
@@ -128,6 +128,13 @@ export function getWorkerTerminalArchive(
|
||||
.get(dispatchId) as WorkerTerminalArchiveRow | undefined
|
||||
}
|
||||
|
||||
/** Existence only; archive rows hold up to 256 KB of output. */
|
||||
export function hasWorkerTerminalArchive(this: OrchestrationDb, dispatchId: string): boolean {
|
||||
return Boolean(
|
||||
this.db.prepare('SELECT 1 FROM worker_terminal_archives WHERE dispatch_id = ?').get(dispatchId)
|
||||
)
|
||||
}
|
||||
|
||||
export function settleWorkerTerminalRelease(
|
||||
this: OrchestrationDb,
|
||||
resourceId: string
|
||||
@@ -284,6 +291,7 @@ export type WorkerTerminalArchiveMethods = {
|
||||
storeWorkerTerminalArchiveBeforeStop: typeof storeWorkerTerminalArchiveBeforeStop
|
||||
commitWorkerTerminalArchiveForRelease: typeof commitWorkerTerminalArchiveForRelease
|
||||
getWorkerTerminalArchive: typeof getWorkerTerminalArchive
|
||||
hasWorkerTerminalArchive: typeof hasWorkerTerminalArchive
|
||||
settleWorkerTerminalRelease: typeof settleWorkerTerminalRelease
|
||||
markWorkerTerminalReleaseUnknown: typeof markWorkerTerminalReleaseUnknown
|
||||
revertWorkerTerminalReleaseToRetained: typeof revertWorkerTerminalReleaseToRetained
|
||||
@@ -297,6 +305,7 @@ export function attachWorkerTerminalArchive(ctor: { prototype: object }): void {
|
||||
storeWorkerTerminalArchiveBeforeStop,
|
||||
commitWorkerTerminalArchiveForRelease,
|
||||
getWorkerTerminalArchive,
|
||||
hasWorkerTerminalArchive,
|
||||
settleWorkerTerminalRelease,
|
||||
markWorkerTerminalReleaseUnknown,
|
||||
revertWorkerTerminalReleaseToRetained,
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
getWorkerAutoCloseState,
|
||||
keepWorkerTerminalOpen
|
||||
} from './worker-terminal-user-keep'
|
||||
import { listWorkerReportWaitCandidates } from './worker-report-wait-candidates'
|
||||
|
||||
/** `databaseId` is the real order key; the timestamp fields only satisfy pre-v3 cursors. */
|
||||
export type WorkerTerminalOrderingKey = {
|
||||
@@ -288,6 +289,7 @@ export type WorkerTerminalListingMethods = {
|
||||
countWorkerTerminalInventory: typeof countWorkerTerminalInventory
|
||||
getWorkerAttentionFacts: typeof getWorkerAttentionFacts
|
||||
getWorkerAttentionFactsForDispatches: typeof getWorkerAttentionFactsForDispatches
|
||||
listWorkerReportWaitCandidates: typeof listWorkerReportWaitCandidates
|
||||
}
|
||||
|
||||
export function attachWorkerTerminalListing(ctor: { prototype: object }): void {
|
||||
@@ -301,6 +303,7 @@ export function attachWorkerTerminalListing(ctor: { prototype: object }): void {
|
||||
getWorkerTerminalOrderingKey,
|
||||
countWorkerTerminalInventory,
|
||||
getWorkerAttentionFacts,
|
||||
getWorkerAttentionFactsForDispatches
|
||||
getWorkerAttentionFactsForDispatches,
|
||||
listWorkerReportWaitCandidates
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import type { FederatedDispatchRow } from '../db/orchestration-records'
|
||||
import { OrchestrationError } from '../orchestration-error'
|
||||
import type { OrchestrationDb } from '../db/orchestration-db'
|
||||
import { FEDERATED_DISPATCH_COLUMN_LIST } from '../db/row-column-lists'
|
||||
|
||||
export function getFederatedDispatch(
|
||||
this: OrchestrationDb,
|
||||
dispatchId: string
|
||||
): FederatedDispatchRow | undefined {
|
||||
return this.db
|
||||
.prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?')
|
||||
.prepare(
|
||||
`SELECT ${FEDERATED_DISPATCH_COLUMN_LIST} FROM federated_dispatches WHERE dispatch_id = ?`
|
||||
)
|
||||
.get(dispatchId) as FederatedDispatchRow | undefined
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { MailboxDeliveryBatch } from '../../../../orchestration/db/messages
|
||||
import { formatMessageBanner } from '../../../../orchestration/mailbox/formatting/message-formatting'
|
||||
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
|
||||
import { completionReportFollowUp } from '../../../../orchestration/workers/reports/report-follow-up-action'
|
||||
import { getMessagesByIds } from '../../../../orchestration/db/messages/messages-by-id'
|
||||
|
||||
// Hide read/sequence, pointer_* and sender_pane_key: they are runtime delivery plumbing, not caller mailbox truth.
|
||||
const INTERNAL_MESSAGE_COLUMNS = [
|
||||
@@ -89,9 +90,14 @@ export function withReportFollowUps<T>(db: OrchestrationDb, result: T): T {
|
||||
const formatted = 'formatted' in result && typeof result.formatted === 'string'
|
||||
let changed = false
|
||||
const lineageCache = new Map()
|
||||
const stored = getMessagesByIds(
|
||||
db,
|
||||
messages
|
||||
.filter((message: MailboxMessageReceipt) => formatted || message.type === 'worker_done')
|
||||
.map((message: MailboxMessageReceipt) => message.id)
|
||||
)
|
||||
const rows = messages.map((message: MailboxMessageReceipt) => {
|
||||
const row =
|
||||
formatted || message.type === 'worker_done' ? db.getMessageById(message.id) : undefined
|
||||
const row = stored.get(message.id)
|
||||
const followUp = row ? completionReportFollowUp(db, row, lineageCache) : undefined
|
||||
changed ||= followUp !== undefined
|
||||
return { message, row, followUp }
|
||||
|
||||
@@ -66,7 +66,11 @@ export function withAnswerNext<T extends object>(receipt: T): T & { next: string
|
||||
/** A worker still runs and worker-list owes no action: each running worker's stop or exit sends mail. */
|
||||
async function awaitsWorkerReports(runtime: OrcaRuntimeService, runId: string): Promise<boolean> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const rows = db.listWorkerTerminalResources({ runId })
|
||||
// Settled history projects to no action, so a long Run's past workers are never projected here.
|
||||
const rows = db.listWorkerTerminalResources({
|
||||
runId,
|
||||
dispatchIds: db.listWorkerReportWaitCandidates(runId)
|
||||
})
|
||||
const { workers } = await projectWorkerListFleet(runtime, rows, {
|
||||
limit: rows.length,
|
||||
completeProjection: true
|
||||
|
||||
@@ -1,16 +1,26 @@
|
||||
import type { WorkerTerminalListState } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
|
||||
|
||||
/** Order by databaseId; keep createdAt/dispatchId so older peers can decode new cursors. */
|
||||
type WorkerListCursorAfter = { createdAt: string; dispatchId: string; databaseId?: number }
|
||||
|
||||
/** The first page's count of the pinned snapshot; later pages repeat it instead of rescanning. */
|
||||
export type WorkerListInventory = {
|
||||
total: number
|
||||
counts: Partial<Record<WorkerTerminalListState, number>>
|
||||
}
|
||||
|
||||
type WorkerListCursorV1 = {
|
||||
version: 1
|
||||
snapshot: { createdAt: string; dispatchId: string }
|
||||
after: WorkerListCursorAfter
|
||||
inventory?: WorkerListInventory
|
||||
}
|
||||
|
||||
type WorkerListCursorV2 = {
|
||||
version: 2
|
||||
snapshot: { databaseId: number }
|
||||
after: WorkerListCursorAfter
|
||||
inventory?: WorkerListInventory
|
||||
}
|
||||
|
||||
type WorkerListCursorV3 = {
|
||||
@@ -51,6 +61,9 @@ export function decodeWorkerListCursor(value: string): WorkerListCursor | null {
|
||||
) {
|
||||
delete parsed.after.databaseId
|
||||
}
|
||||
if ('inventory' in parsed && !isWorkerListInventory(parsed.inventory)) {
|
||||
delete parsed.inventory
|
||||
}
|
||||
if (
|
||||
parsed.version === 1 &&
|
||||
typeof (parsed.snapshot as Partial<WorkerListCursorV1['snapshot']>).createdAt === 'string' &&
|
||||
@@ -76,4 +89,18 @@ export function decodeWorkerListCursor(value: string): WorkerListCursor | null {
|
||||
}
|
||||
}
|
||||
|
||||
function isWorkerListInventory(value: unknown): value is WorkerListInventory {
|
||||
if (typeof value !== 'object' || value === null) {
|
||||
return false
|
||||
}
|
||||
const { total, counts } = value as Partial<WorkerListInventory>
|
||||
return (
|
||||
Number.isSafeInteger(total) &&
|
||||
Number(total) >= 0 &&
|
||||
typeof counts === 'object' &&
|
||||
counts !== null &&
|
||||
Object.values(counts).every((count) => Number.isSafeInteger(count) && Number(count) >= 0)
|
||||
)
|
||||
}
|
||||
|
||||
export type { WorkerListCursor }
|
||||
|
||||
@@ -9,7 +9,8 @@ import { defineOrchestrationMethod } from '../access/orchestration-method-access
|
||||
import {
|
||||
decodeWorkerListCursor,
|
||||
encodeWorkerListCursor,
|
||||
type WorkerListCursor
|
||||
type WorkerListCursor,
|
||||
type WorkerListInventory
|
||||
} from './worker-list-cursor'
|
||||
import {
|
||||
createWorkerListSnapshot,
|
||||
@@ -209,14 +210,16 @@ async function projectWorkerListPageWithFilteredSnapshot(
|
||||
})
|
||||
// Derive totals and counts from the same pinned membership; live filter scans can count rows the cursor cannot reach.
|
||||
const pinnedCount = filteredSnapshot?.dispatchIds.length
|
||||
const inventory =
|
||||
const inventory: WorkerListInventory =
|
||||
pinnedCount !== undefined && params.terminalState
|
||||
? { total: pinnedCount, counts: { [params.terminalState]: pinnedCount } }
|
||||
: db.countWorkerTerminalInventory({
|
||||
runId: params.run,
|
||||
terminalState: params.terminalState,
|
||||
snapshot: args.snapshot
|
||||
})
|
||||
: snapshotCursor?.version !== 3 && snapshotCursor?.inventory
|
||||
? snapshotCursor.inventory
|
||||
: db.countWorkerTerminalInventory({
|
||||
runId: params.run,
|
||||
terminalState: params.terminalState,
|
||||
snapshot: args.snapshot
|
||||
})
|
||||
const nextRow = pageRows.at(-1)
|
||||
fleet.page = {
|
||||
limit,
|
||||
@@ -238,7 +241,8 @@ async function projectWorkerListPageWithFilteredSnapshot(
|
||||
createdAt: nextRow.createdAt,
|
||||
dispatchId: nextRow.dispatchId,
|
||||
databaseId: nextRow.databaseId
|
||||
}
|
||||
},
|
||||
inventory
|
||||
}
|
||||
: {
|
||||
version: 1,
|
||||
@@ -247,7 +251,8 @@ async function projectWorkerListPageWithFilteredSnapshot(
|
||||
createdAt: nextRow.createdAt,
|
||||
dispatchId: nextRow.dispatchId,
|
||||
databaseId: nextRow.databaseId
|
||||
}
|
||||
},
|
||||
inventory
|
||||
}
|
||||
)
|
||||
: null
|
||||
|
||||
@@ -103,6 +103,35 @@ describe('orchestration worker-list pagination', () => {
|
||||
expect(second.counts).toEqual({ retained: 105 })
|
||||
})
|
||||
|
||||
it('repeats the first page counts on later pages of the same snapshot', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
const run = db.createRun({
|
||||
objective: 'Counted once per snapshot',
|
||||
coordinatorHandle: 'term-coordinator',
|
||||
coordinatorPaneKey: 'tab-coordinator:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
})
|
||||
insertDispatch(db, run.id, 'dispatch-a')
|
||||
insertDispatch(db, run.id, 'dispatch-z')
|
||||
|
||||
const first = await callWorkerList(runtime, { run: run.id, limit: 1 })
|
||||
expect(first.counts).toEqual({ retained: 2 })
|
||||
|
||||
sqliteFor(db)
|
||||
.prepare('UPDATE dispatch_contexts SET assignee_handle = NULL WHERE id = ?')
|
||||
.run('dispatch-a')
|
||||
const second = await callWorkerList(runtime, {
|
||||
run: run.id,
|
||||
limit: 1,
|
||||
cursor: first.page.nextCursor
|
||||
})
|
||||
|
||||
expect(second.workers.map((worker) => worker.dispatchId)).toEqual(['dispatch-a'])
|
||||
expect(second.page).toEqual({ total: 2, limit: 1, hasMore: false, nextCursor: null })
|
||||
expect(second.counts).toEqual({ retained: 2 })
|
||||
})
|
||||
|
||||
it('expires a pre-rowid cursor whose anchor row a reset deleted', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = new OrcaRuntimeService()
|
||||
|
||||
@@ -28,7 +28,7 @@ export async function archiveWorkerOutputBeforeStop(args: {
|
||||
if (
|
||||
!resource ||
|
||||
resource.ownership_state !== 'owned' ||
|
||||
db.getWorkerTerminalArchive(dispatchId)
|
||||
db.hasWorkerTerminalArchive(dispatchId)
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ export const DURABLE_EXIT_PROOF = {
|
||||
}
|
||||
|
||||
/** Worker states that carry an outcome; anything else is still supposed to be running. */
|
||||
const SETTLED_WORKER_STATES = new Set(['succeeded', 'failed', 'stopped', 'abandoned'])
|
||||
export const SETTLED_WORKER_STATES = new Set(['succeeded', 'failed', 'stopped', 'abandoned'])
|
||||
|
||||
/** Any certified termination reason proves exit independently of settlement state. */
|
||||
function hasCertifiedExit(worker: FleetLivenessSubject): boolean {
|
||||
|
||||
Reference in New Issue
Block a user