perf(orchestration): stop coordinator check and worker-list from rescanning Run history

- check: the empty-check hint projects only workers that can still be running or owe an action
- worker-list: later pages reuse the first page's inventory counts from the cursor
- check: read completion reports for follow-ups in one query
- outstanding-delivery lookups and the worker seat-grab check use their indexes
- hot by-id lookups name their columns so prepared statements are cached
- stop-archive checks archive existence without reading the archive
This commit is contained in:
Jinwoo-H
2026-11-11 17:59:03 -05:00
parent a44b496f28
commit c7a6766f64
26 changed files with 404 additions and 54 deletions
@@ -15,6 +15,7 @@ import {
import { dispatchAssigneeOrcaSessionId } from '../../addressing/session-lineage/dispatch-assignee-orca-session-id'
import { assertTakesNewWork } from '../worker-terminal/worker-terminal-transfer'
import { retireRemotePlacement } from '../remote-owned/remote-placement-hold'
import { DISPATCH_CONTEXT_COLUMN_LIST } from '../row-column-lists'
export function createDispatchContext(
this: OrchestrationDb,
@@ -126,9 +127,9 @@ export function getDispatchContextById(
this: OrchestrationDb,
dispatchId: string
): DispatchContextRow | undefined {
return this.db.prepare('SELECT * FROM dispatch_contexts WHERE id = ?').get(dispatchId) as
| DispatchContextRow
| undefined
return this.db
.prepare(`SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts WHERE id = ?`)
.get(dispatchId) as DispatchContextRow | undefined
}
export function commitDispatchLaunchTokenHash(
@@ -8,10 +8,11 @@ import { ORCHESTRATION_LEGACY_RUN_ID as LEGACY_RUN_ID } from '../../../../../sha
import { LEGACY_CONTRACT_VERSION } from '../contract-constants'
import { generateId } from '../generated-id'
import type { OrchestrationDb } from '../orchestration-db'
import { LEGACY_ADOPTION_COLUMN_LIST } from '../row-column-lists'
export function getLegacyAdoption(this: OrchestrationDb): LegacyAdoptionRow | undefined {
return this.db
.prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?')
.prepare(`SELECT ${LEGACY_ADOPTION_COLUMN_LIST} FROM legacy_adoptions WHERE source_run_id = ?`)
.get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined
}
@@ -2,6 +2,7 @@ import type { MessageType, MessageRow } from '../orchestration-records'
import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker'
import type { OrchestrationDb } from '../orchestration-db'
import { MESSAGE_COLUMN_LIST } from '../row-column-lists'
const MESSAGE_ID_UPDATE_BATCH_SIZE = 500
const MESSAGE_MUTATION_SAVEPOINT = 'message_id_mutation'
@@ -151,9 +152,9 @@ export function getAllMessages(this: OrchestrationDb, toHandle: string, limit =
}
export function getMessageById(this: OrchestrationDb, id: string): MessageRow | undefined {
const message = this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as
| MessageRow
| undefined
const message = this.db
.prepare(`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id = ?`)
.get(id) as MessageRow | undefined
return message ? exposeMessageTimestamps(message) : undefined
}
@@ -10,6 +10,7 @@ import type { OrchestrationDb } from '../orchestration-db'
import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner'
import { notifyPeerMailQueued } from '../../federation/peer-mail/peer-mail-waiters'
import { ORCHESTRATION_UNBOUND_RUN_ID as UNBOUND_RUN_ID } from '../../../../../shared/orchestration/orchestration-rpc-contract'
import { MESSAGE_COLUMN_LIST } from '../row-column-lists'
const MESSAGE_INSERT_SAVEPOINT = 'message_insert_batch'
const WORKER_DONE_MESSAGE_SAVEPOINT = 'worker_done_message_commit'
@@ -86,7 +87,9 @@ export function insertMessage(this: OrchestrationDb, msg: MessageInsert): Messag
// Mail filed in a peer Run is queued home by trigger in this statement.
notifyPeerMailQueued(this.db)
return exposeMessageTimestamps(
this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow
this.db
.prepare(`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id = ?`)
.get(id) as MessageRow
)
}
@@ -0,0 +1,21 @@
import type { OrchestrationDb } from '../orchestration-db'
import type { MessageRow } from '../orchestration-records'
import { MESSAGE_COLUMN_LIST } from '../row-column-lists'
import { exposeMessageListTimestamps } from '../utc-timestamp'
/** One read for a batch of ids, keyed by id; unknown ids are absent. */
export function getMessagesByIds(
db: OrchestrationDb,
ids: readonly string[]
): Map<string, MessageRow> {
if (ids.length === 0) {
return new Map()
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the column list names every MessageRow column.
const rows = db.db
.prepare(
`SELECT ${MESSAGE_COLUMN_LIST} FROM messages WHERE id IN (SELECT value FROM json_each(?))`
)
.all(JSON.stringify(ids)) as MessageRow[]
return new Map(exposeMessageListTimestamps(rows).map((row) => [row.id, row]))
}
@@ -0,0 +1,28 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from '../orchestration-db'
import { plansOf } from '../query-plan-capture.test-support'
describe('outstanding mailbox Delivery lookups', () => {
let db: OrchestrationDb
beforeEach(() => {
db = new OrchestrationDb(':memory:')
})
afterEach(() => {
db.close()
})
it('read the outstanding index instead of every Delivery ever taken', () => {
const plans = plansOf(db, ['deliveries'], () => {
db.hasOutstandingMailboxDelivery('run:run_a')
db.getOutstandingMailboxDelivery('run:run_a')
db.fenceUnacknowledgedMailboxDeliveries('run:run_a')
})
expect(plans.filter((detail) => detail.startsWith('SCAN deliveries'))).toEqual([])
expect(
plans.filter((detail) => detail.includes('idx_deliveries_one_outstanding'))
).toHaveLength(3)
})
})
@@ -5,9 +5,15 @@ import type { OrchestrationDb } from '../orchestration-db'
import { exposeDeliveryTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
import { requireMailboxConsumer } from './mailbox-consumer'
import { ORCHESTRATION_DELIVERY_BATCH_LIMIT } from './mailbox-routing-page'
import { DELIVERY_COLUMN_LIST } from '../row-column-lists'
// Repeats the outstanding index's predicate so lookups use it; '' marks pre-mailbox rows, never a mailbox.
const OUTSTANDING_IN_MAILBOX = "mailbox_handle = ? AND mailbox_handle != ''"
export function getDeliveryRaw(this: OrchestrationDb, id: string): DeliveryRow | undefined {
return this.db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id) as DeliveryRow | undefined
return this.db.prepare(`SELECT ${DELIVERY_COLUMN_LIST} FROM deliveries WHERE id = ?`).get(id) as
| DeliveryRow
| undefined
}
export function getDeliveryMessages(this: OrchestrationDb, delivery: DeliveryRow): MessageRow[] {
@@ -88,7 +94,7 @@ function selectMailboxDelivery(
ORCHESTRATION_DELIVERY_BATCH_LIMIT
)
const outstanding = db.db
.prepare('SELECT id FROM outstanding_deliveries WHERE mailbox_handle = ?')
.prepare(`SELECT id FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX}`)
.get(params.mailboxHandle)
if (typeof outstanding?.id === 'string') {
const existing = requireDeliveryRaw(db, outstanding.id)
@@ -233,7 +239,7 @@ export function hasOutstandingMailboxDelivery(
): boolean {
return Boolean(
this.db
.prepare('SELECT 1 FROM outstanding_deliveries WHERE mailbox_handle = ? LIMIT 1')
.prepare(`SELECT 1 FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX} LIMIT 1`)
.get(mailboxHandle)
)
}
@@ -243,7 +249,9 @@ export function getOutstandingMailboxDelivery(
mailboxHandle: string
): { id: string; messageIds: string[] } | undefined {
const row = this.db
.prepare('SELECT id, message_ids FROM outstanding_deliveries WHERE mailbox_handle = ? LIMIT 1')
.prepare(
`SELECT id, message_ids FROM outstanding_deliveries WHERE ${OUTSTANDING_IN_MAILBOX} LIMIT 1`
)
.get(mailboxHandle)
if (typeof row?.id !== 'string' || typeof row.message_ids !== 'string') {
return undefined
@@ -260,7 +268,7 @@ export function fenceUnacknowledgedMailboxDeliveries(
): void {
this.db
.prepare(
"UPDATE deliveries SET status = 'fenced' WHERE mailbox_handle = ? AND status = 'outstanding'"
`UPDATE deliveries SET status = 'fenced' WHERE ${OUTSTANDING_IN_MAILBOX} AND status = 'outstanding'`
)
.run(mailboxHandle)
}
@@ -3,6 +3,7 @@ import type { MutationReceiptRow } from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { ensureMutationReceiptCapacity } from './mutation-receipt-capacity'
import type { OrchestrationDb } from '../orchestration-db'
import { MUTATION_RECEIPT_COLUMN_LIST } from '../row-column-lists'
export function getOrCreateLocalMutationCallerFingerprint(this: OrchestrationDb): string {
if (this.localMutationCallerFingerprint) {
@@ -190,7 +191,7 @@ export function getMutationReceipt(
): MutationReceiptRow | undefined {
return this.db
.prepare(
`SELECT * FROM mutation_receipts
`SELECT ${MUTATION_RECEIPT_COLUMN_LIST} FROM mutation_receipts
WHERE caller_fingerprint = ? AND request_id = ?`
)
.get(callerFingerprint, requestId) as MutationReceiptRow | undefined
@@ -4,3 +4,9 @@ import { ORCA_SESSION_ADDRESS_PREFIX } from '../../../../shared/orca-session-add
export function orcaSessionAddressSql(orcaSessionIdSql: string): string {
return `('${ORCA_SESSION_ADDRESS_PREFIX}' || ${orcaSessionIdSql})`
}
/** The bare id a session address names, NULL for anything else; comparing ids lets the id index serve. */
export function orcaSessionIdOfAddressSql(addressSql: string): string {
return `(CASE WHEN substr(${addressSql}, 1, ${ORCA_SESSION_ADDRESS_PREFIX.length}) = '${ORCA_SESSION_ADDRESS_PREFIX}'
THEN substr(${addressSql}, ${ORCA_SESSION_ADDRESS_PREFIX.length + 1}) END)`
}
@@ -0,0 +1,19 @@
import { expect, vi } from 'vitest'
import type { OrchestrationDb } from './orchestration-db'
/** The query plan of every statement `act` prepares that reads or writes one of `tables`. */
export function plansOf(db: OrchestrationDb, tables: string[], act: () => void): string[] {
const prepare = vi.spyOn(db.db, 'prepare')
act()
const statements = prepare.mock.calls
.map(([sql]) => sql)
.filter((sql) => tables.some((table) => sql.includes(table)))
prepare.mockRestore()
expect(statements.length).toBeGreaterThan(0)
return statements.flatMap((sql) =>
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: EXPLAIN QUERY PLAN rows carry a detail column.
(db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]).map(
({ detail }) => detail
)
)
}
@@ -1,25 +1,9 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from '../orchestration-db'
import { plansOf } from '../query-plan-capture.test-support'
import { countRemoteDispatches, listRemoteDispatches } from './remote-dispatch-records'
import { closeRemoteRunLink, openRemoteRunLink } from './remote-run-links'
/** The query plan of every statement `act` prepares that reads or writes one of `tables`. */
function plansOf(db: OrchestrationDb, tables: string[], act: () => void): string[] {
const prepare = vi.spyOn(db.db, 'prepare')
act()
const statements = prepare.mock.calls
.map(([sql]) => sql)
.filter((sql) => tables.some((table) => sql.includes(table)))
prepare.mockRestore()
expect(statements.length).toBeGreaterThan(0)
return statements.flatMap((sql) =>
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: EXPLAIN QUERY PLAN rows carry a detail column.
(db.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all() as { detail: string }[]).map(
({ detail }) => detail
)
)
}
describe('remote-owned queries read through their indexes', () => {
let db: OrchestrationDb
@@ -1,6 +1,11 @@
import type { AttemptObservationStorageRow } from './attempt-observation-store'
import type {
DeliveryRow,
DispatchContextRow,
FederatedDispatchRow,
LegacyAdoptionRow,
MessageRow,
MutationReceiptRow,
RemoteDispatchAttachmentRow,
RunRow,
TaskRow
@@ -147,3 +152,79 @@ export const ATTEMPT_OBSERVATION_FACT_COLUMN_LIST = selectColumns(ATTEMPT_OBSERV
export const REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST = selectColumns(
REMOTE_DISPATCH_ATTACHMENT_COLUMNS
)
/** A projection naming every `Row` column; leaving one out fails compilation. */
function everyColumnOf<Row>() {
return <const Columns extends readonly (keyof Row & string)[]>(
columns: Columns & (Exclude<keyof Row, Columns[number]> extends never ? unknown : never)
): string => selectColumns(columns)
}
export const MESSAGE_COLUMN_LIST = everyColumnOf<MessageRow>()([
'id',
'run_id',
'delivery_contract',
'from_handle',
'to_handle',
'subject',
'body',
'type',
'priority',
'thread_id',
'payload',
'read',
'sequence',
'created_at',
'delivered_at',
'sender_pane_key',
'answer_sender_snapshot',
'pointer_enter_pending',
'pointer_pty_id',
'pointer_process_incarnation'
])
export const DELIVERY_COLUMN_LIST = everyColumnOf<DeliveryRow>()([
'id',
'run_id',
'mailbox_handle',
'consumer_generation',
'message_ids',
'status',
'created_at',
'acknowledged_at'
])
export const MUTATION_RECEIPT_COLUMN_LIST = everyColumnOf<MutationReceiptRow>()([
'caller_fingerprint',
'request_id',
'method',
'payload_hash',
'state',
'receipt',
'created_at',
'updated_at'
])
export const LEGACY_ADOPTION_COLUMN_LIST = everyColumnOf<LegacyAdoptionRow>()([
'source_run_id',
'adopted_run_id',
'scheduler_state_lost',
'adopted_at'
])
export const FEDERATED_DISPATCH_COLUMN_LIST = everyColumnOf<FederatedDispatchRow>()([
'dispatch_id',
'environment_id',
'environment_name',
'peer_fingerprint',
'remote_runtime_epoch',
'protocol_version',
'remote_worktree_id',
'remote_terminal_handle',
'to_home_imported_sequence',
'to_home_acknowledged_sequence',
'home_auto_release_at',
'home_auto_release_state',
'created_at',
'updated_at'
])
@@ -1,7 +1,7 @@
import type Database from '../../../../sqlite/sync-database'
import type { OrchestrationDb } from '../orchestration-db'
import { currentRunCoordinatorSessionAddressSql } from './run-coordinator-orca-session'
import { orcaSessionAddressSql } from '../orca-session-address-sql'
import { orcaSessionAddressSql, orcaSessionIdOfAddressSql } from '../orca-session-address-sql'
/** Mail to an active Dispatch assignee's address in the same Run is that worker's, not coordinator mail. */
export function activeDispatchOwnsAddressSql(runIdSql: string, addressSql: string): string {
@@ -22,16 +22,20 @@ export function activeDispatchOwnsAddressSql(runIdSql: string, addressSql: strin
* because first_bound_at has no milliseconds, so a bind in the second a Dispatch settled is not a grab.
*/
export function workerSeatGrabSql(aliasRow: string): string {
return `EXISTS (
// One EXISTS per address form reads its assignee index; unary + keeps the stat-less planner off the Run index it ties with.
const grabbedAs = (assigneeMatchSql: string) => `EXISTS (
SELECT 1 FROM dispatch_contexts AS grab
WHERE grab.run_id = ${aliasRow}.run_id
AND (grab.assignee_handle = ${aliasRow}.terminal_handle
OR ${orcaSessionAddressSql('grab.assignee_orca_session_id')} = ${aliasRow}.terminal_handle)
WHERE ${assigneeMatchSql}
AND +grab.run_id = ${aliasRow}.run_id
AND NOT (grab.creator_handle IS NOT NULL AND grab.creator_handle = grab.assignee_handle)
AND julianday(grab.created_at) <= julianday(${aliasRow}.first_bound_at)
AND (grab.status IN ('pending', 'dispatched')
OR julianday(datetime(grab.completed_at)) > julianday(${aliasRow}.first_bound_at))
)`
return `(${grabbedAs(`grab.assignee_handle = ${aliasRow}.terminal_handle`)}
OR ${grabbedAs(
`grab.assignee_orca_session_id = ${orcaSessionIdOfAddressSql(`${aliasRow}.terminal_handle`)}`
)})`
}
/** Whether `addressSql` is a coordinator address of the Run `runIdSql`, seat grabs excluded. */
@@ -0,0 +1,29 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from '../orchestration-db'
import { plansOf } from '../query-plan-capture.test-support'
describe('worker seat-grab check', () => {
let db: OrchestrationDb
beforeEach(() => {
db = new OrchestrationDb(':memory:')
})
afterEach(() => {
db.close()
})
it("reads the address's own Dispatches, not every Dispatch in the Run", () => {
const plans = plansOf(db, ['dispatch_contexts'], () =>
db.getRunMailboxOwnerIdsForHandle('term_coordinator')
)
expect(plans.filter((detail) => detail.includes('idx_dispatch_run_status'))).toEqual([])
expect(plans).toEqual(
expect.arrayContaining([
expect.stringContaining('idx_dispatch_assignee_handle'),
expect.stringContaining('idx_dispatch_assignee_orca_session_id')
])
)
})
})
@@ -17,7 +17,8 @@ export const DERIVED_DELIVERY_SCHEMA_SQL = `
CREATE TRIGGER IF NOT EXISTS trg_deliveries_one_outstanding
AFTER INSERT ON deliveries
WHEN NEW.mailbox_handle != '' AND EXISTS (
SELECT 1 FROM outstanding_deliveries WHERE mailbox_handle = NEW.mailbox_handle LIMIT 1 OFFSET 1
SELECT 1 FROM outstanding_deliveries
WHERE mailbox_handle = NEW.mailbox_handle AND mailbox_handle != '' LIMIT 1 OFFSET 1
)
BEGIN
SELECT RAISE(ABORT, 'Mailbox already has an outstanding delivery');
@@ -0,0 +1,76 @@
import { fleetWorkerIdentity } from '../../../../../shared/orchestration/fleet/orchestration-fleet-agent-status-evidence'
import { SETTLED_WORKER_STATES } from '../../../../../shared/orchestration/fleet/orchestration-fleet-worker-projection'
import type { OrchestrationDb } from '../orchestration-db'
import { WORKER_TERMINAL_HANDED_ON_SQL } from './worker-terminal-inventory-counts'
const SETTLED_WORKER_STATE_LIST = [...SETTLED_WORKER_STATES].map((state) => `'${state}'`).join(', ')
type CandidateRow = {
dispatch_id: string
agent_terminal_handle: string | null
pane_key: string | null
may_run: number
may_owe_action: number
}
/** The keys worker-list's status index counts owners by, from the same identity. */
function ownerKeys(row: CandidateRow): string[] {
const identity = fleetWorkerIdentity({
agentTerminalHandle: row.agent_terminal_handle,
paneKey: row.pane_key
})
if (identity.kind === 'unidentifiable') {
return []
}
const handleKey = `handle:${identity.terminalHandle}`
return identity.kind === 'pane_and_terminal'
? [handleKey, `pane:${identity.paneKey}`]
: [handleKey]
}
/**
* The Run's Dispatches whose worker-list outcome can be in_progress or whose next action can be
* other than none, plus one other owner of each handle or pane they use, since a shared identity
* withholds status evidence. Everything else is settled history that projects to none.
*/
export function listWorkerReportWaitCandidates(this: OrchestrationDb, runId: string): string[] {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the query selects exactly these columns.
const rows = this.db
.prepare(
`SELECT d.id AS dispatch_id,
CASE WHEN ${WORKER_TERMINAL_HANDED_ON_SQL} THEN NULL
ELSE COALESCE(w.agent_terminal_handle, d.assignee_handle) END AS agent_terminal_handle,
CASE WHEN ${WORKER_TERMINAL_HANDED_ON_SQL} THEN NULL
ELSE COALESCE(r.pane_key, d.assignee_pane_key) END AS pane_key,
(d.status IN ('pending', 'dispatched')
OR EXISTS (SELECT 1 FROM attempt_observation_facts f
WHERE f.dispatch_id = d.id AND f.facet = 'liveness')) AS may_run,
((COALESCE(w.state, 'unsupervised') NOT IN (${SETTLED_WORKER_STATE_LIST})
AND w.stage IS NOT 'released' AND r.release_state IS NOT 'released'
AND NOT (d.status = 'completed' AND w.dispatch_id IS NULL AND r.id IS NULL))
OR (r.ownership_state = 'owned' AND r.release_state IS NOT 'released'
AND r.release_state IS NOT 'retained')
) AS may_owe_action
FROM dispatch_contexts d
LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id
LEFT JOIN worker_terminal_resources r ON r.owner_dispatch_id = d.id
WHERE d.run_id = ?`
)
.all(runId) as CandidateRow[]
// Without a possibly running worker the hint cannot apply, so nothing needs projecting.
if (!rows.some((row) => row.may_run === 1)) {
return []
}
const isCandidate = (row: CandidateRow) => row.may_run === 1 || row.may_owe_action === 1
const kept = rows.filter(isCandidate)
// One more owner already makes a key shared; further owners change nothing.
const unshared = new Set(kept.flatMap(ownerKeys))
for (const row of rows) {
const claimed = isCandidate(row) ? [] : ownerKeys(row).filter((key) => unshared.has(key))
if (claimed.length > 0) {
claimed.forEach((key) => unshared.delete(key))
kept.push(row)
}
}
return [...new Set(kept.map((row) => row.dispatch_id))]
}
@@ -128,6 +128,13 @@ export function getWorkerTerminalArchive(
.get(dispatchId) as WorkerTerminalArchiveRow | undefined
}
/** Existence only; archive rows hold up to 256 KB of output. */
export function hasWorkerTerminalArchive(this: OrchestrationDb, dispatchId: string): boolean {
return Boolean(
this.db.prepare('SELECT 1 FROM worker_terminal_archives WHERE dispatch_id = ?').get(dispatchId)
)
}
export function settleWorkerTerminalRelease(
this: OrchestrationDb,
resourceId: string
@@ -284,6 +291,7 @@ export type WorkerTerminalArchiveMethods = {
storeWorkerTerminalArchiveBeforeStop: typeof storeWorkerTerminalArchiveBeforeStop
commitWorkerTerminalArchiveForRelease: typeof commitWorkerTerminalArchiveForRelease
getWorkerTerminalArchive: typeof getWorkerTerminalArchive
hasWorkerTerminalArchive: typeof hasWorkerTerminalArchive
settleWorkerTerminalRelease: typeof settleWorkerTerminalRelease
markWorkerTerminalReleaseUnknown: typeof markWorkerTerminalReleaseUnknown
revertWorkerTerminalReleaseToRetained: typeof revertWorkerTerminalReleaseToRetained
@@ -297,6 +305,7 @@ export function attachWorkerTerminalArchive(ctor: { prototype: object }): void {
storeWorkerTerminalArchiveBeforeStop,
commitWorkerTerminalArchiveForRelease,
getWorkerTerminalArchive,
hasWorkerTerminalArchive,
settleWorkerTerminalRelease,
markWorkerTerminalReleaseUnknown,
revertWorkerTerminalReleaseToRetained,
@@ -23,6 +23,7 @@ import {
getWorkerAutoCloseState,
keepWorkerTerminalOpen
} from './worker-terminal-user-keep'
import { listWorkerReportWaitCandidates } from './worker-report-wait-candidates'
/** `databaseId` is the real order key; the timestamp fields only satisfy pre-v3 cursors. */
export type WorkerTerminalOrderingKey = {
@@ -288,6 +289,7 @@ export type WorkerTerminalListingMethods = {
countWorkerTerminalInventory: typeof countWorkerTerminalInventory
getWorkerAttentionFacts: typeof getWorkerAttentionFacts
getWorkerAttentionFactsForDispatches: typeof getWorkerAttentionFactsForDispatches
listWorkerReportWaitCandidates: typeof listWorkerReportWaitCandidates
}
export function attachWorkerTerminalListing(ctor: { prototype: object }): void {
@@ -301,6 +303,7 @@ export function attachWorkerTerminalListing(ctor: { prototype: object }): void {
getWorkerTerminalOrderingKey,
countWorkerTerminalInventory,
getWorkerAttentionFacts,
getWorkerAttentionFactsForDispatches
getWorkerAttentionFactsForDispatches,
listWorkerReportWaitCandidates
})
}
@@ -1,13 +1,16 @@
import type { FederatedDispatchRow } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
import { FEDERATED_DISPATCH_COLUMN_LIST } from '../db/row-column-lists'
export function getFederatedDispatch(
this: OrchestrationDb,
dispatchId: string
): FederatedDispatchRow | undefined {
return this.db
.prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?')
.prepare(
`SELECT ${FEDERATED_DISPATCH_COLUMN_LIST} FROM federated_dispatches WHERE dispatch_id = ?`
)
.get(dispatchId) as FederatedDispatchRow | undefined
}
@@ -3,6 +3,7 @@ import type { MailboxDeliveryBatch } from '../../../../orchestration/db/messages
import { formatMessageBanner } from '../../../../orchestration/mailbox/formatting/message-formatting'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { completionReportFollowUp } from '../../../../orchestration/workers/reports/report-follow-up-action'
import { getMessagesByIds } from '../../../../orchestration/db/messages/messages-by-id'
// Hide read/sequence, pointer_* and sender_pane_key: they are runtime delivery plumbing, not caller mailbox truth.
const INTERNAL_MESSAGE_COLUMNS = [
@@ -89,9 +90,14 @@ export function withReportFollowUps<T>(db: OrchestrationDb, result: T): T {
const formatted = 'formatted' in result && typeof result.formatted === 'string'
let changed = false
const lineageCache = new Map()
const stored = getMessagesByIds(
db,
messages
.filter((message: MailboxMessageReceipt) => formatted || message.type === 'worker_done')
.map((message: MailboxMessageReceipt) => message.id)
)
const rows = messages.map((message: MailboxMessageReceipt) => {
const row =
formatted || message.type === 'worker_done' ? db.getMessageById(message.id) : undefined
const row = stored.get(message.id)
const followUp = row ? completionReportFollowUp(db, row, lineageCache) : undefined
changed ||= followUp !== undefined
return { message, row, followUp }
@@ -66,7 +66,11 @@ export function withAnswerNext<T extends object>(receipt: T): T & { next: string
/** A worker still runs and worker-list owes no action: each running worker's stop or exit sends mail. */
async function awaitsWorkerReports(runtime: OrcaRuntimeService, runId: string): Promise<boolean> {
const db = runtime.getOrchestrationDb()
const rows = db.listWorkerTerminalResources({ runId })
// Settled history projects to no action, so a long Run's past workers are never projected here.
const rows = db.listWorkerTerminalResources({
runId,
dispatchIds: db.listWorkerReportWaitCandidates(runId)
})
const { workers } = await projectWorkerListFleet(runtime, rows, {
limit: rows.length,
completeProjection: true
@@ -1,16 +1,26 @@
import type { WorkerTerminalListState } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
/** Order by databaseId; keep createdAt/dispatchId so older peers can decode new cursors. */
type WorkerListCursorAfter = { createdAt: string; dispatchId: string; databaseId?: number }
/** The first page's count of the pinned snapshot; later pages repeat it instead of rescanning. */
export type WorkerListInventory = {
total: number
counts: Partial<Record<WorkerTerminalListState, number>>
}
type WorkerListCursorV1 = {
version: 1
snapshot: { createdAt: string; dispatchId: string }
after: WorkerListCursorAfter
inventory?: WorkerListInventory
}
type WorkerListCursorV2 = {
version: 2
snapshot: { databaseId: number }
after: WorkerListCursorAfter
inventory?: WorkerListInventory
}
type WorkerListCursorV3 = {
@@ -51,6 +61,9 @@ export function decodeWorkerListCursor(value: string): WorkerListCursor | null {
) {
delete parsed.after.databaseId
}
if ('inventory' in parsed && !isWorkerListInventory(parsed.inventory)) {
delete parsed.inventory
}
if (
parsed.version === 1 &&
typeof (parsed.snapshot as Partial<WorkerListCursorV1['snapshot']>).createdAt === 'string' &&
@@ -76,4 +89,18 @@ export function decodeWorkerListCursor(value: string): WorkerListCursor | null {
}
}
function isWorkerListInventory(value: unknown): value is WorkerListInventory {
if (typeof value !== 'object' || value === null) {
return false
}
const { total, counts } = value as Partial<WorkerListInventory>
return (
Number.isSafeInteger(total) &&
Number(total) >= 0 &&
typeof counts === 'object' &&
counts !== null &&
Object.values(counts).every((count) => Number.isSafeInteger(count) && Number(count) >= 0)
)
}
export type { WorkerListCursor }
@@ -9,7 +9,8 @@ import { defineOrchestrationMethod } from '../access/orchestration-method-access
import {
decodeWorkerListCursor,
encodeWorkerListCursor,
type WorkerListCursor
type WorkerListCursor,
type WorkerListInventory
} from './worker-list-cursor'
import {
createWorkerListSnapshot,
@@ -209,14 +210,16 @@ async function projectWorkerListPageWithFilteredSnapshot(
})
// Derive totals and counts from the same pinned membership; live filter scans can count rows the cursor cannot reach.
const pinnedCount = filteredSnapshot?.dispatchIds.length
const inventory =
const inventory: WorkerListInventory =
pinnedCount !== undefined && params.terminalState
? { total: pinnedCount, counts: { [params.terminalState]: pinnedCount } }
: db.countWorkerTerminalInventory({
runId: params.run,
terminalState: params.terminalState,
snapshot: args.snapshot
})
: snapshotCursor?.version !== 3 && snapshotCursor?.inventory
? snapshotCursor.inventory
: db.countWorkerTerminalInventory({
runId: params.run,
terminalState: params.terminalState,
snapshot: args.snapshot
})
const nextRow = pageRows.at(-1)
fleet.page = {
limit,
@@ -238,7 +241,8 @@ async function projectWorkerListPageWithFilteredSnapshot(
createdAt: nextRow.createdAt,
dispatchId: nextRow.dispatchId,
databaseId: nextRow.databaseId
}
},
inventory
}
: {
version: 1,
@@ -247,7 +251,8 @@ async function projectWorkerListPageWithFilteredSnapshot(
createdAt: nextRow.createdAt,
dispatchId: nextRow.dispatchId,
databaseId: nextRow.databaseId
}
},
inventory
}
)
: null
@@ -103,6 +103,35 @@ describe('orchestration worker-list pagination', () => {
expect(second.counts).toEqual({ retained: 105 })
})
it('repeats the first page counts on later pages of the same snapshot', async () => {
db = new OrchestrationDb(':memory:')
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
const run = db.createRun({
objective: 'Counted once per snapshot',
coordinatorHandle: 'term-coordinator',
coordinatorPaneKey: 'tab-coordinator:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
})
insertDispatch(db, run.id, 'dispatch-a')
insertDispatch(db, run.id, 'dispatch-z')
const first = await callWorkerList(runtime, { run: run.id, limit: 1 })
expect(first.counts).toEqual({ retained: 2 })
sqliteFor(db)
.prepare('UPDATE dispatch_contexts SET assignee_handle = NULL WHERE id = ?')
.run('dispatch-a')
const second = await callWorkerList(runtime, {
run: run.id,
limit: 1,
cursor: first.page.nextCursor
})
expect(second.workers.map((worker) => worker.dispatchId)).toEqual(['dispatch-a'])
expect(second.page).toEqual({ total: 2, limit: 1, hasMore: false, nextCursor: null })
expect(second.counts).toEqual({ retained: 2 })
})
it('expires a pre-rowid cursor whose anchor row a reset deleted', async () => {
db = new OrchestrationDb(':memory:')
const runtime = new OrcaRuntimeService()
@@ -28,7 +28,7 @@ export async function archiveWorkerOutputBeforeStop(args: {
if (
!resource ||
resource.ownership_state !== 'owned' ||
db.getWorkerTerminalArchive(dispatchId)
db.hasWorkerTerminalArchive(dispatchId)
) {
return
}
@@ -40,7 +40,7 @@ export const DURABLE_EXIT_PROOF = {
}
/** Worker states that carry an outcome; anything else is still supposed to be running. */
const SETTLED_WORKER_STATES = new Set(['succeeded', 'failed', 'stopped', 'abandoned'])
export const SETTLED_WORKER_STATES = new Set(['succeeded', 'failed', 'stopped', 'abandoned'])
/** Any certified termination reason proves exit independently of settlement state. */
function hasCertifiedExit(worker: FleetLivenessSubject): boolean {