From c7a6766f64ea0d53d4a4e8f90fc509a60efc892e Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Wed, 11 Nov 2026 17:59:03 -0500 Subject: [PATCH] perf(orchestration): stop coordinator check and worker-list from rescanning Run history - check: the empty-check hint projects only workers that can still be running or owe an action - worker-list: later pages reuse the first page's inventory counts from the cursor - check: read completion reports for follow-ups in one query - outstanding-delivery lookups and the worker seat-grab check use their indexes - hot by-id lookups name their columns so prepared statements are cached - stop-archive checks archive existence without reading the archive --- .../dispatch-context-store.ts | 7 +- .../legacy/legacy-compatibility-principals.ts | 3 +- .../db/messages/message-inbox.ts | 7 +- .../db/messages/message-insert.ts | 5 +- .../db/messages/messages-by-id.ts | 21 +++++ .../outstanding-delivery-query-plans.test.ts | 28 +++++++ .../db/messages/role-mailbox-delivery.ts | 18 +++-- .../mutation-receipt-store.ts | 3 +- .../db/orca-session-address-sql.ts | 6 ++ .../db/query-plan-capture.test-support.ts | 19 +++++ .../remote-owned-query-plans.test.ts | 20 +---- .../orchestration/db/row-column-lists.ts | 81 +++++++++++++++++++ .../db/runs/run-coordinator-mail-routing.ts | 14 ++-- .../runs/worker-seat-grab-query-plans.test.ts | 29 +++++++ .../orchestration/db/schema/migrate-v41.ts | 3 +- .../worker-report-wait-candidates.ts | 76 +++++++++++++++++ .../worker-terminal-archive.ts | 9 +++ .../worker-terminal-listing.ts | 5 +- .../federated-dispatch-store.ts | 5 +- .../messaging/mailbox-message-receipt.ts | 10 ++- .../rpc/methods/orchestration/receipt-next.ts | 6 +- .../worker/worker-list-cursor.ts | 27 +++++++ .../worker/worker-list-method.ts | 23 +++--- .../worker/worker-list-pagination.test.ts | 29 +++++++ .../worker/worker-stop-archive.ts | 2 +- .../orchestration-fleet-worker-projection.ts | 2 +- 26 files changed, 404 insertions(+), 54 deletions(-) create mode 100644 src/main/runtime/orchestration/db/messages/messages-by-id.ts create mode 100644 src/main/runtime/orchestration/db/messages/outstanding-delivery-query-plans.test.ts create mode 100644 src/main/runtime/orchestration/db/query-plan-capture.test-support.ts create mode 100644 src/main/runtime/orchestration/db/runs/worker-seat-grab-query-plans.test.ts create mode 100644 src/main/runtime/orchestration/db/worker-terminal/worker-report-wait-candidates.ts diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts index 9e4a7900871..1093d5fe6a8 100644 --- a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts @@ -15,6 +15,7 @@ import { import { dispatchAssigneeOrcaSessionId } from '../../addressing/session-lineage/dispatch-assignee-orca-session-id' import { assertTakesNewWork } from '../worker-terminal/worker-terminal-transfer' import { retireRemotePlacement } from '../remote-owned/remote-placement-hold' +import { DISPATCH_CONTEXT_COLUMN_LIST } from '../row-column-lists' export function createDispatchContext( this: OrchestrationDb, @@ -126,9 +127,9 @@ export function getDispatchContextById( this: OrchestrationDb, dispatchId: string ): DispatchContextRow | undefined { - return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(dispatchId) as - | DispatchContextRow - | undefined + return this.db + .prepare(`SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts WHERE id = ?`) + .get(dispatchId) as DispatchContextRow | undefined } export function commitDispatchLaunchTokenHash( diff --git a/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts b/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts index 508af1b2541..e14505e4100 100644 --- a/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts +++ b/src/main/runtime/orchestration/db/legacy/legacy-compatibility-principals.ts @@ -8,10 +8,11 @@ import { ORCHESTRATION_LEGACY_RUN_ID as LEGACY_RUN_ID } from '../../../../../sha import { LEGACY_CONTRACT_VERSION } from '../contract-constants' import { generateId } from '../generated-id' import type { OrchestrationDb } from '../orchestration-db' +import { LEGACY_ADOPTION_COLUMN_LIST } from '../row-column-lists' export function getLegacyAdoption(this: OrchestrationDb): LegacyAdoptionRow | undefined { return this.db - .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') + .prepare(`SELECT ${LEGACY_ADOPTION_COLUMN_LIST} FROM legacy_adoptions WHERE source_run_id = ?`) .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined } diff --git a/src/main/runtime/orchestration/db/messages/message-inbox.ts b/src/main/runtime/orchestration/db/messages/message-inbox.ts index f201d19e808..1b56a1f1bb4 100644 --- a/src/main/runtime/orchestration/db/messages/message-inbox.ts +++ b/src/main/runtime/orchestration/db/messages/message-inbox.ts @@ -2,6 +2,7 @@ import type { MessageType, MessageRow } from '../orchestration-records' import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp' import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker' import type { OrchestrationDb } from '../orchestration-db' +import { MESSAGE_COLUMN_LIST } from '../row-column-lists' const MESSAGE_ID_UPDATE_BATCH_SIZE = 500 const MESSAGE_MUTATION_SAVEPOINT = 'message_id_mutation' @@ -151,9 +152,9 @@ export function getAllMessages(this: OrchestrationDb, toHandle: string, limit = } export function getMessageById(this: OrchestrationDb, id: string): MessageRow | undefined { - const message = this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as - | MessageRow - | undefined + const message = this.db + .prepare(`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id = ?`) + .get(id) as MessageRow | undefined return message ? exposeMessageTimestamps(message) : undefined } diff --git a/src/main/runtime/orchestration/db/messages/message-insert.ts b/src/main/runtime/orchestration/db/messages/message-insert.ts index 24c45a19e7d..82695105ca1 100644 --- a/src/main/runtime/orchestration/db/messages/message-insert.ts +++ b/src/main/runtime/orchestration/db/messages/message-insert.ts @@ -10,6 +10,7 @@ import type { OrchestrationDb } from '../orchestration-db' import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner' import { notifyPeerMailQueued } from '../../federation/peer-mail/peer-mail-waiters' import { ORCHESTRATION_UNBOUND_RUN_ID as UNBOUND_RUN_ID } from '../../../../../shared/orchestration/orchestration-rpc-contract' +import { MESSAGE_COLUMN_LIST } from '../row-column-lists' const MESSAGE_INSERT_SAVEPOINT = 'message_insert_batch' const WORKER_DONE_MESSAGE_SAVEPOINT = 'worker_done_message_commit' @@ -86,7 +87,9 @@ export function insertMessage(this: OrchestrationDb, msg: MessageInsert): Messag // Mail filed in a peer Run is queued home by trigger in this statement. notifyPeerMailQueued(this.db) return exposeMessageTimestamps( - this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow + this.db + .prepare(`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id = ?`) + .get(id) as MessageRow ) } diff --git a/src/main/runtime/orchestration/db/messages/messages-by-id.ts b/src/main/runtime/orchestration/db/messages/messages-by-id.ts new file mode 100644 index 00000000000..297c20b22ea --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/messages-by-id.ts @@ -0,0 +1,21 @@ +import type { OrchestrationDb } from '../orchestration-db' +import type { MessageRow } from '../orchestration-records' +import { MESSAGE_COLUMN_LIST } from '../row-column-lists' +import { exposeMessageListTimestamps } from '../utc-timestamp' + +/** One read for a batch of ids, keyed by id; unknown ids are absent. */ +export function getMessagesByIds( + db: OrchestrationDb, + ids: readonly string[] +): Map { + if (ids.length === 0) { + return new Map() + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the column list names every MessageRow column. + const rows = db.db + .prepare( + `SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id IN (SELECT value FROM json_each(?))` + ) + .all(JSON.stringify(ids)) as MessageRow[] + return new Map(exposeMessageListTimestamps(rows).map((row) => [row.id, row])) +} diff --git a/src/main/runtime/orchestration/db/messages/outstanding-delivery-query-plans.test.ts b/src/main/runtime/orchestration/db/messages/outstanding-delivery-query-plans.test.ts new file mode 100644 index 00000000000..32f7b6059b0 --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/outstanding-delivery-query-plans.test.ts @@ -0,0 +1,28 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from '../orchestration-db' +import { plansOf } from '../query-plan-capture.test-support' + +describe('outstanding mailbox Delivery lookups', () => { + let db: OrchestrationDb + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + }) + + afterEach(() => { + db.close() + }) + + it('read the outstanding index instead of every Delivery ever taken', () => { + const plans = plansOf(db, ['deliveries'], () => { + db.hasOutstandingMailboxDelivery('run:run_a') + db.getOutstandingMailboxDelivery('run:run_a') + db.fenceUnacknowledgedMailboxDeliveries('run:run_a') + }) + + expect(plans.filter((detail) => detail.startsWith('SCAN deliveries'))).toEqual([]) + expect( + plans.filter((detail) => detail.includes('idx_deliveries_one_outstanding')) + ).toHaveLength(3) + }) +}) diff --git a/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts b/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts index a8c2e381900..c4e764924cd 100644 --- a/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts +++ b/src/main/runtime/orchestration/db/messages/role-mailbox-delivery.ts @@ -5,9 +5,15 @@ import type { OrchestrationDb } from '../orchestration-db' import { exposeDeliveryTimestamps, exposeMessageListTimestamps } from '../utc-timestamp' import { requireMailboxConsumer } from './mailbox-consumer' import { ORCHESTRATION_DELIVERY_BATCH_LIMIT } from './mailbox-routing-page' +import { DELIVERY_COLUMN_LIST } from '../row-column-lists' + +// Repeats the outstanding index's predicate so lookups use it; '' marks pre-mailbox rows, never a mailbox. +const OUTSTANDING_IN_MAILBOX = "mailbox_handle = ? AND mailbox_handle != ''" export function getDeliveryRaw(this: OrchestrationDb, id: string): DeliveryRow | undefined { - return this.db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id) as DeliveryRow | undefined + return this.db.prepare(`SELECT ${DELIVERY_COLUMN_LIST} FROM deliveries WHERE id = ?`).get(id) as + | DeliveryRow + | undefined } export function getDeliveryMessages(this: OrchestrationDb, delivery: DeliveryRow): MessageRow[] { @@ -88,7 +94,7 @@ function selectMailboxDelivery( ORCHESTRATION_DELIVERY_BATCH_LIMIT ) const outstanding = db.db - .prepare('SELECT id FROM outstanding_deliveries WHERE mailbox_handle = ?') + .prepare(`SELECT id FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX}`) .get(params.mailboxHandle) if (typeof outstanding?.id === 'string') { const existing = requireDeliveryRaw(db, outstanding.id) @@ -233,7 +239,7 @@ export function hasOutstandingMailboxDelivery( ): boolean { return Boolean( this.db - .prepare('SELECT 1 FROM outstanding_deliveries WHERE mailbox_handle = ? LIMIT 1') + .prepare(`SELECT 1 FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX} LIMIT 1`) .get(mailboxHandle) ) } @@ -243,7 +249,9 @@ export function getOutstandingMailboxDelivery( mailboxHandle: string ): { id: string; messageIds: string[] } | undefined { const row = this.db - .prepare('SELECT id, message_ids FROM outstanding_deliveries WHERE mailbox_handle = ? LIMIT 1') + .prepare( + `SELECT id, message_ids FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX} LIMIT 1` + ) .get(mailboxHandle) if (typeof row?.id !== 'string' || typeof row.message_ids !== 'string') { return undefined @@ -260,7 +268,7 @@ export function fenceUnacknowledgedMailboxDeliveries( ): void { this.db .prepare( - "UPDATE deliveries SET status = 'fenced' WHERE mailbox_handle = ? AND status = 'outstanding'" + `UPDATE deliveries SET status = 'fenced' WHERE ${OUTSTANDING_IN_MAILBOX} AND status = 'outstanding'` ) .run(mailboxHandle) } diff --git a/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts b/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts index 874b9a1b091..1b0c8e83611 100644 --- a/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts +++ b/src/main/runtime/orchestration/db/mutation-receipts/mutation-receipt-store.ts @@ -3,6 +3,7 @@ import type { MutationReceiptRow } from '../orchestration-records' import { OrchestrationError } from '../../orchestration-error' import { ensureMutationReceiptCapacity } from './mutation-receipt-capacity' import type { OrchestrationDb } from '../orchestration-db' +import { MUTATION_RECEIPT_COLUMN_LIST } from '../row-column-lists' export function getOrCreateLocalMutationCallerFingerprint(this: OrchestrationDb): string { if (this.localMutationCallerFingerprint) { @@ -190,7 +191,7 @@ export function getMutationReceipt( ): MutationReceiptRow | undefined { return this.db .prepare( - `SELECT * FROM mutation_receipts + `SELECT ${MUTATION_RECEIPT_COLUMN_LIST} FROM mutation_receipts WHERE caller_fingerprint = ? AND request_id = ?` ) .get(callerFingerprint, requestId) as MutationReceiptRow | undefined diff --git a/src/main/runtime/orchestration/db/orca-session-address-sql.ts b/src/main/runtime/orchestration/db/orca-session-address-sql.ts index ee1f849f1ad..9f805eb1c1e 100644 --- a/src/main/runtime/orchestration/db/orca-session-address-sql.ts +++ b/src/main/runtime/orchestration/db/orca-session-address-sql.ts @@ -4,3 +4,9 @@ import { ORCA_SESSION_ADDRESS_PREFIX } from '../../../../shared/orca-session-add export function orcaSessionAddressSql(orcaSessionIdSql: string): string { return `('${ORCA_SESSION_ADDRESS_PREFIX}' || ${orcaSessionIdSql})` } + +/** The bare id a session address names, NULL for anything else; comparing ids lets the id index serve. */ +export function orcaSessionIdOfAddressSql(addressSql: string): string { + return `(CASE WHEN substr(${addressSql}, 1, ${ORCA_SESSION_ADDRESS_PREFIX.length}) = '${ORCA_SESSION_ADDRESS_PREFIX}' + THEN substr(${addressSql}, ${ORCA_SESSION_ADDRESS_PREFIX.length + 1}) END)` +} diff --git a/src/main/runtime/orchestration/db/query-plan-capture.test-support.ts b/src/main/runtime/orchestration/db/query-plan-capture.test-support.ts new file mode 100644 index 00000000000..e662471dc6c --- /dev/null +++ b/src/main/runtime/orchestration/db/query-plan-capture.test-support.ts @@ -0,0 +1,19 @@ +import { expect, vi } from 'vitest' +import type { OrchestrationDb } from './orchestration-db' + +/** The query plan of every statement `act` prepares that reads or writes one of `tables`. */ +export function plansOf(db: OrchestrationDb, tables: string[], act: () => void): string[] { + const prepare = vi.spyOn(db.db, 'prepare') + act() + const statements = prepare.mock.calls + .map(([sql]) => sql) + .filter((sql) => tables.some((table) => sql.includes(table))) + prepare.mockRestore() + expect(statements.length).toBeGreaterThan(0) + return statements.flatMap((sql) => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: EXPLAIN QUERY PLAN rows carry a detail column. + (db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]).map( + ({ detail }) => detail + ) + ) +} diff --git a/src/main/runtime/orchestration/db/remote-owned/remote-owned-query-plans.test.ts b/src/main/runtime/orchestration/db/remote-owned/remote-owned-query-plans.test.ts index f33bdec0662..658538355db 100644 --- a/src/main/runtime/orchestration/db/remote-owned/remote-owned-query-plans.test.ts +++ b/src/main/runtime/orchestration/db/remote-owned/remote-owned-query-plans.test.ts @@ -1,25 +1,9 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { OrchestrationDb } from '../orchestration-db' +import { plansOf } from '../query-plan-capture.test-support' import { countRemoteDispatches, listRemoteDispatches } from './remote-dispatch-records' import { closeRemoteRunLink, openRemoteRunLink } from './remote-run-links' -/** The query plan of every statement `act` prepares that reads or writes one of `tables`. */ -function plansOf(db: OrchestrationDb, tables: string[], act: () => void): string[] { - const prepare = vi.spyOn(db.db, 'prepare') - act() - const statements = prepare.mock.calls - .map(([sql]) => sql) - .filter((sql) => tables.some((table) => sql.includes(table))) - prepare.mockRestore() - expect(statements.length).toBeGreaterThan(0) - return statements.flatMap((sql) => - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: EXPLAIN QUERY PLAN rows carry a detail column. - (db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]).map( - ({ detail }) => detail - ) - ) -} - describe('remote-owned queries read through their indexes', () => { let db: OrchestrationDb diff --git a/src/main/runtime/orchestration/db/row-column-lists.ts b/src/main/runtime/orchestration/db/row-column-lists.ts index 350f2d721f5..287841fb329 100644 --- a/src/main/runtime/orchestration/db/row-column-lists.ts +++ b/src/main/runtime/orchestration/db/row-column-lists.ts @@ -1,6 +1,11 @@ import type { AttemptObservationStorageRow } from './attempt-observation-store' import type { + DeliveryRow, DispatchContextRow, + FederatedDispatchRow, + LegacyAdoptionRow, + MessageRow, + MutationReceiptRow, RemoteDispatchAttachmentRow, RunRow, TaskRow @@ -147,3 +152,79 @@ export const ATTEMPT_OBSERVATION_FACT_COLUMN_LIST = selectColumns(ATTEMPT_OBSERV export const REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST = selectColumns( REMOTE_DISPATCH_ATTACHMENT_COLUMNS ) + +/** A projection naming every `Row` column; leaving one out fails compilation. */ +function everyColumnOf() { + return ( + columns: Columns & (Exclude extends never ? unknown : never) + ): string => selectColumns(columns) +} + +export const MESSAGE_COLUMN_LIST = everyColumnOf()([ + 'id', + 'run_id', + 'delivery_contract', + 'from_handle', + 'to_handle', + 'subject', + 'body', + 'type', + 'priority', + 'thread_id', + 'payload', + 'read', + 'sequence', + 'created_at', + 'delivered_at', + 'sender_pane_key', + 'answer_sender_snapshot', + 'pointer_enter_pending', + 'pointer_pty_id', + 'pointer_process_incarnation' +]) + +export const DELIVERY_COLUMN_LIST = everyColumnOf()([ + 'id', + 'run_id', + 'mailbox_handle', + 'consumer_generation', + 'message_ids', + 'status', + 'created_at', + 'acknowledged_at' +]) + +export const MUTATION_RECEIPT_COLUMN_LIST = everyColumnOf()([ + 'caller_fingerprint', + 'request_id', + 'method', + 'payload_hash', + 'state', + 'receipt', + 'created_at', + 'updated_at' +]) + +export const LEGACY_ADOPTION_COLUMN_LIST = everyColumnOf()([ + 'source_run_id', + 'adopted_run_id', + 'scheduler_state_lost', + 'adopted_at' +]) + +export const FEDERATED_DISPATCH_COLUMN_LIST = everyColumnOf()([ + 'dispatch_id', + 'environment_id', + 'environment_name', + 'peer_fingerprint', + 'remote_runtime_epoch', + 'protocol_version', + 'remote_worktree_id', + 'remote_terminal_handle', + 'to_home_imported_sequence', + 'to_home_acknowledged_sequence', + 'home_auto_release_at', + 'home_auto_release_state', + 'created_at', + 'updated_at' +]) diff --git a/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts b/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts index 8640a9964fd..48876ff1790 100644 --- a/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts +++ b/src/main/runtime/orchestration/db/runs/run-coordinator-mail-routing.ts @@ -1,7 +1,7 @@ import type Database from '../../../../sqlite/sync-database' import type { OrchestrationDb } from '../orchestration-db' import { currentRunCoordinatorSessionAddressSql } from './run-coordinator-orca-session' -import { orcaSessionAddressSql } from '../orca-session-address-sql' +import { orcaSessionAddressSql, orcaSessionIdOfAddressSql } from '../orca-session-address-sql' /** Mail to an active Dispatch assignee's address in the same Run is that worker's, not coordinator mail. */ export function activeDispatchOwnsAddressSql(runIdSql: string, addressSql: string): string { @@ -22,16 +22,20 @@ export function activeDispatchOwnsAddressSql(runIdSql: string, addressSql: strin * because first_bound_at has no milliseconds, so a bind in the second a Dispatch settled is not a grab. */ export function workerSeatGrabSql(aliasRow: string): string { - return `EXISTS ( + // One EXISTS per address form reads its assignee index; unary + keeps the stat-less planner off the Run index it ties with. + const grabbedAs = (assigneeMatchSql: string) => `EXISTS ( SELECT 1 FROM dispatch_contexts AS grab - WHERE grab.run_id = ${aliasRow}.run_id - AND (grab.assignee_handle = ${aliasRow}.terminal_handle - OR ${orcaSessionAddressSql('grab.assignee_orca_session_id')} = ${aliasRow}.terminal_handle) + WHERE ${assigneeMatchSql} + AND +grab.run_id = ${aliasRow}.run_id AND NOT (grab.creator_handle IS NOT NULL AND grab.creator_handle = grab.assignee_handle) AND julianday(grab.created_at) <= julianday(${aliasRow}.first_bound_at) AND (grab.status IN ('pending', 'dispatched') OR julianday(datetime(grab.completed_at)) > julianday(${aliasRow}.first_bound_at)) )` + return `(${grabbedAs(`grab.assignee_handle = ${aliasRow}.terminal_handle`)} + OR ${grabbedAs( + `grab.assignee_orca_session_id = ${orcaSessionIdOfAddressSql(`${aliasRow}.terminal_handle`)}` + )})` } /** Whether `addressSql` is a coordinator address of the Run `runIdSql`, seat grabs excluded. */ diff --git a/src/main/runtime/orchestration/db/runs/worker-seat-grab-query-plans.test.ts b/src/main/runtime/orchestration/db/runs/worker-seat-grab-query-plans.test.ts new file mode 100644 index 00000000000..66ad312dd00 --- /dev/null +++ b/src/main/runtime/orchestration/db/runs/worker-seat-grab-query-plans.test.ts @@ -0,0 +1,29 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from '../orchestration-db' +import { plansOf } from '../query-plan-capture.test-support' + +describe('worker seat-grab check', () => { + let db: OrchestrationDb + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + }) + + afterEach(() => { + db.close() + }) + + it("reads the address's own Dispatches, not every Dispatch in the Run", () => { + const plans = plansOf(db, ['dispatch_contexts'], () => + db.getRunMailboxOwnerIdsForHandle('term_coordinator') + ) + + expect(plans.filter((detail) => detail.includes('idx_dispatch_run_status'))).toEqual([]) + expect(plans).toEqual( + expect.arrayContaining([ + expect.stringContaining('idx_dispatch_assignee_handle'), + expect.stringContaining('idx_dispatch_assignee_orca_session_id') + ]) + ) + }) +}) diff --git a/src/main/runtime/orchestration/db/schema/migrate-v41.ts b/src/main/runtime/orchestration/db/schema/migrate-v41.ts index 6f7fc909e6d..098b3616f27 100644 --- a/src/main/runtime/orchestration/db/schema/migrate-v41.ts +++ b/src/main/runtime/orchestration/db/schema/migrate-v41.ts @@ -17,7 +17,8 @@ export const DERIVED_DELIVERY_SCHEMA_SQL = ` CREATE TRIGGER IF NOT EXISTS trg_deliveries_one_outstanding AFTER INSERT ON deliveries WHEN NEW.mailbox_handle != '' AND EXISTS ( - SELECT 1 FROM outstanding_deliveries WHERE mailbox_handle = NEW.mailbox_handle LIMIT 1 OFFSET 1 + SELECT 1 FROM outstanding_deliveries + WHERE mailbox_handle = NEW.mailbox_handle AND mailbox_handle != '' LIMIT 1 OFFSET 1 ) BEGIN SELECT RAISE(ABORT, 'Mailbox already has an outstanding delivery'); diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-report-wait-candidates.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-report-wait-candidates.ts new file mode 100644 index 00000000000..9f76e1162a6 --- /dev/null +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-report-wait-candidates.ts @@ -0,0 +1,76 @@ +import { fleetWorkerIdentity } from '../../../../../shared/orchestration/fleet/orchestration-fleet-agent-status-evidence' +import { SETTLED_WORKER_STATES } from '../../../../../shared/orchestration/fleet/orchestration-fleet-worker-projection' +import type { OrchestrationDb } from '../orchestration-db' +import { WORKER_TERMINAL_HANDED_ON_SQL } from './worker-terminal-inventory-counts' + +const SETTLED_WORKER_STATE_LIST = [...SETTLED_WORKER_STATES].map((state) => `'${state}'`).join(', ') + +type CandidateRow = { + dispatch_id: string + agent_terminal_handle: string | null + pane_key: string | null + may_run: number + may_owe_action: number +} + +/** The keys worker-list's status index counts owners by, from the same identity. */ +function ownerKeys(row: CandidateRow): string[] { + const identity = fleetWorkerIdentity({ + agentTerminalHandle: row.agent_terminal_handle, + paneKey: row.pane_key + }) + if (identity.kind === 'unidentifiable') { + return [] + } + const handleKey = `handle:${identity.terminalHandle}` + return identity.kind === 'pane_and_terminal' + ? [handleKey, `pane:${identity.paneKey}`] + : [handleKey] +} + +/** + * The Run's Dispatches whose worker-list outcome can be in_progress or whose next action can be + * other than none, plus one other owner of each handle or pane they use, since a shared identity + * withholds status evidence. Everything else is settled history that projects to none. + */ +export function listWorkerReportWaitCandidates(this: OrchestrationDb, runId: string): string[] { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the query selects exactly these columns. + const rows = this.db + .prepare( + `SELECT d.id AS dispatch_id, + CASE WHEN ${WORKER_TERMINAL_HANDED_ON_SQL} THEN NULL + ELSE COALESCE(w.agent_terminal_handle, d.assignee_handle) END AS agent_terminal_handle, + CASE WHEN ${WORKER_TERMINAL_HANDED_ON_SQL} THEN NULL + ELSE COALESCE(r.pane_key, d.assignee_pane_key) END AS pane_key, + (d.status IN ('pending', 'dispatched') + OR EXISTS (SELECT 1 FROM attempt_observation_facts f + WHERE f.dispatch_id = d.id AND f.facet = 'liveness')) AS may_run, + ((COALESCE(w.state, 'unsupervised') NOT IN (${SETTLED_WORKER_STATE_LIST}) + AND w.stage IS NOT 'released' AND r.release_state IS NOT 'released' + AND NOT (d.status = 'completed' AND w.dispatch_id IS NULL AND r.id IS NULL)) + OR (r.ownership_state = 'owned' AND r.release_state IS NOT 'released' + AND r.release_state IS NOT 'retained') + ) AS may_owe_action + FROM dispatch_contexts d + LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id + LEFT JOIN worker_terminal_resources r ON r.owner_dispatch_id = d.id + WHERE d.run_id = ?` + ) + .all(runId) as CandidateRow[] + // Without a possibly running worker the hint cannot apply, so nothing needs projecting. + if (!rows.some((row) => row.may_run === 1)) { + return [] + } + const isCandidate = (row: CandidateRow) => row.may_run === 1 || row.may_owe_action === 1 + const kept = rows.filter(isCandidate) + // One more owner already makes a key shared; further owners change nothing. + const unshared = new Set(kept.flatMap(ownerKeys)) + for (const row of rows) { + const claimed = isCandidate(row) ? [] : ownerKeys(row).filter((key) => unshared.has(key)) + if (claimed.length > 0) { + claimed.forEach((key) => unshared.delete(key)) + kept.push(row) + } + } + return [...new Set(kept.map((row) => row.dispatch_id))] +} diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts index 6f622df2c59..6f932cdefdd 100644 --- a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-archive.ts @@ -128,6 +128,13 @@ export function getWorkerTerminalArchive( .get(dispatchId) as WorkerTerminalArchiveRow | undefined } +/** Existence only; archive rows hold up to 256 KB of output. */ +export function hasWorkerTerminalArchive(this: OrchestrationDb, dispatchId: string): boolean { + return Boolean( + this.db.prepare('SELECT 1 FROM worker_terminal_archives WHERE dispatch_id = ?').get(dispatchId) + ) +} + export function settleWorkerTerminalRelease( this: OrchestrationDb, resourceId: string @@ -284,6 +291,7 @@ export type WorkerTerminalArchiveMethods = { storeWorkerTerminalArchiveBeforeStop: typeof storeWorkerTerminalArchiveBeforeStop commitWorkerTerminalArchiveForRelease: typeof commitWorkerTerminalArchiveForRelease getWorkerTerminalArchive: typeof getWorkerTerminalArchive + hasWorkerTerminalArchive: typeof hasWorkerTerminalArchive settleWorkerTerminalRelease: typeof settleWorkerTerminalRelease markWorkerTerminalReleaseUnknown: typeof markWorkerTerminalReleaseUnknown revertWorkerTerminalReleaseToRetained: typeof revertWorkerTerminalReleaseToRetained @@ -297,6 +305,7 @@ export function attachWorkerTerminalArchive(ctor: { prototype: object }): void { storeWorkerTerminalArchiveBeforeStop, commitWorkerTerminalArchiveForRelease, getWorkerTerminalArchive, + hasWorkerTerminalArchive, settleWorkerTerminalRelease, markWorkerTerminalReleaseUnknown, revertWorkerTerminalReleaseToRetained, diff --git a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts index c2ce51ded5e..d7f8f38ce03 100644 --- a/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts +++ b/src/main/runtime/orchestration/db/worker-terminal/worker-terminal-listing.ts @@ -23,6 +23,7 @@ import { getWorkerAutoCloseState, keepWorkerTerminalOpen } from './worker-terminal-user-keep' +import { listWorkerReportWaitCandidates } from './worker-report-wait-candidates' /** `databaseId` is the real order key; the timestamp fields only satisfy pre-v3 cursors. */ export type WorkerTerminalOrderingKey = { @@ -288,6 +289,7 @@ export type WorkerTerminalListingMethods = { countWorkerTerminalInventory: typeof countWorkerTerminalInventory getWorkerAttentionFacts: typeof getWorkerAttentionFacts getWorkerAttentionFactsForDispatches: typeof getWorkerAttentionFactsForDispatches + listWorkerReportWaitCandidates: typeof listWorkerReportWaitCandidates } export function attachWorkerTerminalListing(ctor: { prototype: object }): void { @@ -301,6 +303,7 @@ export function attachWorkerTerminalListing(ctor: { prototype: object }): void { getWorkerTerminalOrderingKey, countWorkerTerminalInventory, getWorkerAttentionFacts, - getWorkerAttentionFactsForDispatches + getWorkerAttentionFactsForDispatches, + listWorkerReportWaitCandidates }) } diff --git a/src/main/runtime/orchestration/legacy-remote-home/federated-dispatch-store.ts b/src/main/runtime/orchestration/legacy-remote-home/federated-dispatch-store.ts index f2cf2a85589..1152878d0f1 100644 --- a/src/main/runtime/orchestration/legacy-remote-home/federated-dispatch-store.ts +++ b/src/main/runtime/orchestration/legacy-remote-home/federated-dispatch-store.ts @@ -1,13 +1,16 @@ import type { FederatedDispatchRow } from '../db/orchestration-records' import { OrchestrationError } from '../orchestration-error' import type { OrchestrationDb } from '../db/orchestration-db' +import { FEDERATED_DISPATCH_COLUMN_LIST } from '../db/row-column-lists' export function getFederatedDispatch( this: OrchestrationDb, dispatchId: string ): FederatedDispatchRow | undefined { return this.db - .prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?') + .prepare( + `SELECT ${FEDERATED_DISPATCH_COLUMN_LIST} FROM federated_dispatches WHERE dispatch_id = ?` + ) .get(dispatchId) as FederatedDispatchRow | undefined } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/mailbox-message-receipt.ts b/src/main/runtime/rpc/methods/orchestration/messaging/mailbox-message-receipt.ts index 4561130732c..6b30619e317 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/mailbox-message-receipt.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/mailbox-message-receipt.ts @@ -3,6 +3,7 @@ import type { MailboxDeliveryBatch } from '../../../../orchestration/db/messages import { formatMessageBanner } from '../../../../orchestration/mailbox/formatting/message-formatting' import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' import { completionReportFollowUp } from '../../../../orchestration/workers/reports/report-follow-up-action' +import { getMessagesByIds } from '../../../../orchestration/db/messages/messages-by-id' // Hide read/sequence, pointer_* and sender_pane_key: they are runtime delivery plumbing, not caller mailbox truth. const INTERNAL_MESSAGE_COLUMNS = [ @@ -89,9 +90,14 @@ export function withReportFollowUps(db: OrchestrationDb, result: T): T { const formatted = 'formatted' in result && typeof result.formatted === 'string' let changed = false const lineageCache = new Map() + const stored = getMessagesByIds( + db, + messages + .filter((message: MailboxMessageReceipt) => formatted || message.type === 'worker_done') + .map((message: MailboxMessageReceipt) => message.id) + ) const rows = messages.map((message: MailboxMessageReceipt) => { - const row = - formatted || message.type === 'worker_done' ? db.getMessageById(message.id) : undefined + const row = stored.get(message.id) const followUp = row ? completionReportFollowUp(db, row, lineageCache) : undefined changed ||= followUp !== undefined return { message, row, followUp } diff --git a/src/main/runtime/rpc/methods/orchestration/receipt-next.ts b/src/main/runtime/rpc/methods/orchestration/receipt-next.ts index 0b7dca55f3d..fcbd5f02784 100644 --- a/src/main/runtime/rpc/methods/orchestration/receipt-next.ts +++ b/src/main/runtime/rpc/methods/orchestration/receipt-next.ts @@ -66,7 +66,11 @@ export function withAnswerNext(receipt: T): T & { next: string /** A worker still runs and worker-list owes no action: each running worker's stop or exit sends mail. */ async function awaitsWorkerReports(runtime: OrcaRuntimeService, runId: string): Promise { const db = runtime.getOrchestrationDb() - const rows = db.listWorkerTerminalResources({ runId }) + // Settled history projects to no action, so a long Run's past workers are never projected here. + const rows = db.listWorkerTerminalResources({ + runId, + dispatchIds: db.listWorkerReportWaitCandidates(runId) + }) const { workers } = await projectWorkerListFleet(runtime, rows, { limit: rows.length, completeProjection: true diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-cursor.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-cursor.ts index df3284989b0..b87fdcec2c8 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-cursor.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-cursor.ts @@ -1,16 +1,26 @@ +import type { WorkerTerminalListState } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership' + /** Order by databaseId; keep createdAt/dispatchId so older peers can decode new cursors. */ type WorkerListCursorAfter = { createdAt: string; dispatchId: string; databaseId?: number } +/** The first page's count of the pinned snapshot; later pages repeat it instead of rescanning. */ +export type WorkerListInventory = { + total: number + counts: Partial> +} + type WorkerListCursorV1 = { version: 1 snapshot: { createdAt: string; dispatchId: string } after: WorkerListCursorAfter + inventory?: WorkerListInventory } type WorkerListCursorV2 = { version: 2 snapshot: { databaseId: number } after: WorkerListCursorAfter + inventory?: WorkerListInventory } type WorkerListCursorV3 = { @@ -51,6 +61,9 @@ export function decodeWorkerListCursor(value: string): WorkerListCursor | null { ) { delete parsed.after.databaseId } + if ('inventory' in parsed && !isWorkerListInventory(parsed.inventory)) { + delete parsed.inventory + } if ( parsed.version === 1 && typeof (parsed.snapshot as Partial).createdAt === 'string' && @@ -76,4 +89,18 @@ export function decodeWorkerListCursor(value: string): WorkerListCursor | null { } } +function isWorkerListInventory(value: unknown): value is WorkerListInventory { + if (typeof value !== 'object' || value === null) { + return false + } + const { total, counts } = value as Partial + return ( + Number.isSafeInteger(total) && + Number(total) >= 0 && + typeof counts === 'object' && + counts !== null && + Object.values(counts).every((count) => Number.isSafeInteger(count) && Number(count) >= 0) + ) +} + export type { WorkerListCursor } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts index 958da8fee3c..7cd0f2d9c63 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts @@ -9,7 +9,8 @@ import { defineOrchestrationMethod } from '../access/orchestration-method-access import { decodeWorkerListCursor, encodeWorkerListCursor, - type WorkerListCursor + type WorkerListCursor, + type WorkerListInventory } from './worker-list-cursor' import { createWorkerListSnapshot, @@ -209,14 +210,16 @@ async function projectWorkerListPageWithFilteredSnapshot( }) // Derive totals and counts from the same pinned membership; live filter scans can count rows the cursor cannot reach. const pinnedCount = filteredSnapshot?.dispatchIds.length - const inventory = + const inventory: WorkerListInventory = pinnedCount !== undefined && params.terminalState ? { total: pinnedCount, counts: { [params.terminalState]: pinnedCount } } - : db.countWorkerTerminalInventory({ - runId: params.run, - terminalState: params.terminalState, - snapshot: args.snapshot - }) + : snapshotCursor?.version !== 3 && snapshotCursor?.inventory + ? snapshotCursor.inventory + : db.countWorkerTerminalInventory({ + runId: params.run, + terminalState: params.terminalState, + snapshot: args.snapshot + }) const nextRow = pageRows.at(-1) fleet.page = { limit, @@ -238,7 +241,8 @@ async function projectWorkerListPageWithFilteredSnapshot( createdAt: nextRow.createdAt, dispatchId: nextRow.dispatchId, databaseId: nextRow.databaseId - } + }, + inventory } : { version: 1, @@ -247,7 +251,8 @@ async function projectWorkerListPageWithFilteredSnapshot( createdAt: nextRow.createdAt, dispatchId: nextRow.dispatchId, databaseId: nextRow.databaseId - } + }, + inventory } ) : null diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-pagination.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-pagination.test.ts index 0bae1e3b3ee..5a2f24acffb 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-pagination.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-pagination.test.ts @@ -103,6 +103,35 @@ describe('orchestration worker-list pagination', () => { expect(second.counts).toEqual({ retained: 105 }) }) + it('repeats the first page counts on later pages of the same snapshot', async () => { + db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + const run = db.createRun({ + objective: 'Counted once per snapshot', + coordinatorHandle: 'term-coordinator', + coordinatorPaneKey: 'tab-coordinator:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + insertDispatch(db, run.id, 'dispatch-a') + insertDispatch(db, run.id, 'dispatch-z') + + const first = await callWorkerList(runtime, { run: run.id, limit: 1 }) + expect(first.counts).toEqual({ retained: 2 }) + + sqliteFor(db) + .prepare('UPDATE dispatch_contexts SET assignee_handle = NULL WHERE id = ?') + .run('dispatch-a') + const second = await callWorkerList(runtime, { + run: run.id, + limit: 1, + cursor: first.page.nextCursor + }) + + expect(second.workers.map((worker) => worker.dispatchId)).toEqual(['dispatch-a']) + expect(second.page).toEqual({ total: 2, limit: 1, hasMore: false, nextCursor: null }) + expect(second.counts).toEqual({ retained: 2 }) + }) + it('expires a pre-rowid cursor whose anchor row a reset deleted', async () => { db = new OrchestrationDb(':memory:') const runtime = new OrcaRuntimeService() diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-stop-archive.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-stop-archive.ts index 891731ed8af..4ecf136e7e5 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-stop-archive.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-stop-archive.ts @@ -28,7 +28,7 @@ export async function archiveWorkerOutputBeforeStop(args: { if ( !resource || resource.ownership_state !== 'owned' || - db.getWorkerTerminalArchive(dispatchId) + db.hasWorkerTerminalArchive(dispatchId) ) { return } diff --git a/src/shared/orchestration/fleet/orchestration-fleet-worker-projection.ts b/src/shared/orchestration/fleet/orchestration-fleet-worker-projection.ts index b33211dc61e..3b932146249 100644 --- a/src/shared/orchestration/fleet/orchestration-fleet-worker-projection.ts +++ b/src/shared/orchestration/fleet/orchestration-fleet-worker-projection.ts @@ -40,7 +40,7 @@ export const DURABLE_EXIT_PROOF = { } /** Worker states that carry an outcome; anything else is still supposed to be running. */ -const SETTLED_WORKER_STATES = new Set(['succeeded', 'failed', 'stopped', 'abandoned']) +export const SETTLED_WORKER_STATES = new Set(['succeeded', 'failed', 'stopped', 'abandoned']) /** Any certified termination reason proves exit independently of settlement state. */ function hasCertifiedExit(worker: FleetLivenessSubject): boolean {