From 0d696aef33484cda8faf1f299ddaa2735c6f77df Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:42:50 -0500 Subject: [PATCH] Let a finished worker keep messaging its coordinator (#38357) * Let a finished worker keep messaging its coordinator * Tell a finished remote worker why its coordinator is out of reach * Let a finished worker's ask wait for its answer instead of starting a follow-up * Share the assignee check, keep the finished lookup in dispatch-lookup, and fix review findings * Match a finished terminal worker by its process too; cover resume after a follow-up --- src/cli/specs/orchestration.ts | 4 +- .../db/dispatch-context/dispatch-lookup.ts | 99 ++++----- .../worker-report-settlement.ts | 9 +- .../remote-dispatch-attachment-stop.ts | 28 ++- .../db/questions/question-threads.ts | 9 +- .../orchestration/worker-report-admission.ts | 15 +- .../orchestration/messaging/ask-methods.ts | 66 ++++-- .../messaging/finished-worker-mail.test.ts | 189 ++++++++++++++++++ .../messaging/finished-worker-mail.ts | 101 ++++++++++ .../orchestration/messaging/send-methods.ts | 25 ++- .../rpc/methods/orchestration/routing.ts | 12 +- 11 files changed, 456 insertions(+), 101 deletions(-) create mode 100644 src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index c507da2fd47..5de2e60ef2a 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -79,7 +79,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ 'On Windows PowerShell, quote group addresses such as --to "@all" or --to "@worktree:".', "worker_done and heartbeat are exact-Dispatch signals and cannot target groups; omit --to to use the Dispatch's Run mailbox.", 'worker_done requires --outcome succeeded or --outcome failed.', - 'From an active Dispatch, an omitted recipient defaults to its owning Run mailbox.', + 'From a worker, an omitted recipient defaults to its coordinator, also after it finished; mail sent after finishing, worker_done included, leaves the finished task unchanged.', 'Use --to dispatch: for attempt-specific coordinator guidance; Orca durably relays it to a connected worker server.', 'A worker_done with the active task/dispatch IDs completes that task only from the dispatched pane. When stable pane identity is unavailable, the sender handle must exactly match the dispatch assignee; injected preambles include the correct --from value.', 'Prefer --task-id/--dispatch-id/etc. over raw --payload JSON in worker commands; PowerShell strips JSON quotes easily.' @@ -239,7 +239,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ ], identityFlagRoles: { from: 'caller' }, notes: [ - 'From an active Dispatch, a new question goes to its owning Run mailbox and ask returns at once: end your turn, and Orca wakes you with the answer as mail. --wait and --resume block until the answer instead; --timeout-ms accepts 1 to 1800000 ms per wait (default 600000 = 10 minutes, max 30 minutes), and a timeout leaves the question pending.' + 'From a worker, a new question goes to its coordinator and ask returns at once: end your turn, and Orca wakes you with the answer as mail. A worker that already finished waits for the answer instead. --wait and --resume block until the answer instead; --timeout-ms accepts 1 to 1800000 ms per wait (default 600000 = 10 minutes, max 30 minutes), and a timeout leaves the question pending.' ] }, { diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts index 75e55e731f3..f67883c7869 100644 --- a/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-lookup.ts @@ -13,21 +13,22 @@ import { equivalentDispatchMailboxOwners } from '../dispatch-mailbox-selection' -// Hoist explicit projections so graph-publish queries hit the statement cache. -const ACTIVE_DISPATCH_BY_HANDLE_SQL = - // Newest-first avoids selecting a stale row when one handle has multiple active Dispatches. +// Hoist explicit projections so graph-publish queries hit the statement cache; newest first, since +// one handle can hold several active Dispatches. +const assigneeSql = (where: string, status: string) => `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts - WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') - ORDER BY rowid DESC LIMIT 1` -const ACTIVE_DISPATCH_BY_PANE_KEY_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts - WHERE assignee_pane_key = ? AND status IN ('pending', 'dispatched') - ORDER BY rowid DESC LIMIT 1` -const ACTIVE_DISPATCH_BY_PANE_SUFFIX_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts - WHERE assignee_pane_key IS NOT NULL - AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1 - AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ? - ORDER BY rowid DESC LIMIT 1` -const LATEST_DISPATCH_BY_HANDLE_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1` + WHERE ${where} ${status} ORDER BY rowid DESC LIMIT 1` +const BY_PANE_SUFFIX = `assignee_pane_key IS NOT NULL AND instr(assignee_pane_key, ':') > 1 + AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?` +const assigneeLookups = (status: string) => ({ + byHandle: assigneeSql('assignee_handle = ?', status), + byPaneKey: assigneeSql('assignee_pane_key = ?', status), + byPaneSuffix: assigneeSql(BY_PANE_SUFFIX, status) +}) +const ASSIGNEE_SQL = { + active: assigneeLookups("AND status IN ('pending', 'dispatched')"), + latest: assigneeLookups('') +} function presentationAddressKey(column: string, address: string | null): string { return `${column}:${address}` @@ -232,16 +233,23 @@ export function isDispatchMessageSender( if (params.allowCanonicalDispatchHandle && params.handle === `dispatch:${dispatch.id}`) { return true } + return isDispatchAssignee(dispatch, params.handle, params.paneKey) +} + +/** Whether this handle or pane was given the Dispatch, whatever its status. */ +export function isDispatchAssignee( + dispatch: Pick, + handle: string, + paneKey: string | null | undefined +): boolean { if ( - params.paneKey && + paneKey && dispatch.assignee_pane_key && - isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) + isEquivalentPaneKey(dispatch.assignee_pane_key, paneKey) ) { return true } - return ( - params.handle === dispatch.assignee_handle && (!params.paneKey || !dispatch.assignee_pane_key) - ) + return handle === dispatch.assignee_handle && (!paneKey || !dispatch.assignee_pane_key) } export function findActiveDispatchForAssignee( @@ -249,38 +257,37 @@ export function findActiveDispatchForAssignee( assigneeHandle: string, assigneePaneKey?: string ): DispatchContextRow | undefined { - const byHandle = this.db.prepare(ACTIVE_DISPATCH_BY_HANDLE_SQL).get(assigneeHandle) as - | DispatchContextRow - | undefined - if (byHandle) { - return byHandle - } - - if (!assigneePaneKey) { - return undefined - } - - const exactPane = this.db.prepare(ACTIVE_DISPATCH_BY_PANE_KEY_SQL).get(assigneePaneKey) as - | DispatchContextRow - | undefined - if (exactPane) { - return exactPane - } - if (!parsePaneKey(assigneePaneKey)) { - return undefined - } - return this.db - .prepare(ACTIVE_DISPATCH_BY_PANE_SUFFIX_SQL) - .get(paneKeyMatchSuffix(assigneePaneKey)) as DispatchContextRow | undefined + return findDispatchForAssignee(this, 'active', assigneeHandle, assigneePaneKey) } +/** The newest Dispatch, settled or not; with a pane, also after the handle was reminted. */ export function getLatestDispatchForTerminal( this: OrchestrationDb, - handle: string + handle: string, + paneKey?: string ): DispatchContextRow | undefined { - return this.db.prepare(LATEST_DISPATCH_BY_HANDLE_SQL).get(handle) as - | DispatchContextRow - | undefined + return findDispatchForAssignee(this, 'latest', handle, paneKey) +} + +function findDispatchForAssignee( + db: OrchestrationDb, + kind: keyof typeof ASSIGNEE_SQL, + handle: string, + paneKey: string | undefined +): DispatchContextRow | undefined { + const sql = ASSIGNEE_SQL[kind] + const read = (statement: string, value: string) => + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: each statement selects the schema-pinned Dispatch projection. + db.db.prepare(statement).get(value) as DispatchContextRow | undefined + const byHandle = read(sql.byHandle, handle) + if (byHandle || !paneKey) { + return byHandle + } + const exactPane = read(sql.byPaneKey, paneKey) + if (exactPane || !parsePaneKey(paneKey)) { + return exactPane + } + return read(sql.byPaneSuffix, paneKeyMatchSuffix(paneKey)) } /** Helpers this Dispatch started that have not settled; their reports wake their creator. */ diff --git a/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts b/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts index f945c90bef8..980586fa4de 100644 --- a/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts +++ b/src/main/runtime/orchestration/db/dispatch-context/worker-report-settlement.ts @@ -1,13 +1,13 @@ import { markUnreadFollowUpMail } from './worker-follow-up-lineage' import type { WorkerReportOutcome, WorkerReportSettlement } from '../../types' import type { OrchestrationDb } from '../orchestration-db' -import { AGENT_PROMPT_STALLED_ERROR } from '../../../agent-prompt-submission-verification' import { settleActiveDispatchesForTask } from './dispatch-completion' import { getActiveDispatchForTask } from './task-dispatch-reconciliation' import { transitionLifecycleWithDb } from '../lifecycle-transition' import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner' import { isWorkerStateIn, + reportCorrectsUnobservedPrompt, SETTLEABLE_WORKER_STATES, UNPROVEN_WORKER_STATES } from '../../worker-report-admission' @@ -93,11 +93,8 @@ export function settleWorkerReportInTransaction( const expectedDispatchStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' const expectedTaskStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' - // A first-hand worker report can correct an unobserved prompt failure, even after circuit break; check before duplicate detection (#16095). - const settledByUnobservedPrompt = - (dispatch.status === 'failed' || dispatch.status === 'circuit_broken') && - dispatch.last_failure === AGENT_PROMPT_STALLED_ERROR && - task.status === 'failed' + // Check before duplicate detection (#16095). + const settledByUnobservedPrompt = reportCorrectsUnobservedPrompt(dispatch, task) const reportingWorker = this.getWorkerDispatch(params.dispatchId) const settledDispatchStatus = dispatch.status === 'circuit_broken' ? 'failed' : dispatch.status if ( diff --git a/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts index a39cc00febd..9dad41c9b68 100644 --- a/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts +++ b/src/main/runtime/orchestration/db/federation/remote-dispatch-attachment-stop.ts @@ -73,22 +73,38 @@ export function markRemoteAttachmentStopUnknown( export function findActiveRemoteAttachmentForPane( this: OrchestrationDb, paneKey: string +): RemoteDispatchAttachmentRow | undefined { + return findRemoteAttachmentForPane(this, paneKey, `${potentiallyLiveRemoteAttachmentSql()} AND`) +} + +/** The newest remote Dispatch this pane ran, finished or not. */ +export function findLatestRemoteAttachmentForPane( + this: OrchestrationDb, + paneKey: string +): RemoteDispatchAttachmentRow | undefined { + return findRemoteAttachmentForPane(this, paneKey, '') +} + +function findRemoteAttachmentForPane( + db: OrchestrationDb, + paneKey: string, + stateFilter: string ): RemoteDispatchAttachmentRow | undefined { if (!parsePaneKey(paneKey)) { // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The schema-pinned attachment projection returns RemoteDispatchAttachmentRow rows; the adapter exposes unknown. - return this.db + return db.db .prepare( `SELECT ${REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST} FROM remote_dispatch_attachments - WHERE ${potentiallyLiveRemoteAttachmentSql()} AND pane_key = ? + WHERE ${stateFilter} pane_key = ? ORDER BY rowid DESC LIMIT 1` ) .get(paneKey) as RemoteDispatchAttachmentRow | undefined } // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The schema-pinned attachment projection returns RemoteDispatchAttachmentRow rows; the adapter exposes unknown. - return this.db + return db.db .prepare( `SELECT ${REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST} FROM remote_dispatch_attachments - WHERE ${potentiallyLiveRemoteAttachmentSql()} AND pane_key IS NOT NULL + WHERE ${stateFilter} pane_key IS NOT NULL AND instr(pane_key, ':') > 1 AND ${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL} = ? ORDER BY rowid DESC LIMIT 1` @@ -101,6 +117,7 @@ export type RemoteDispatchAttachmentStopMethods = { settleRemoteAttachmentStop: typeof settleRemoteAttachmentStop markRemoteAttachmentStopUnknown: typeof markRemoteAttachmentStopUnknown findActiveRemoteAttachmentForPane: typeof findActiveRemoteAttachmentForPane + findLatestRemoteAttachmentForPane: typeof findLatestRemoteAttachmentForPane } export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }): void { @@ -108,6 +125,7 @@ export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }): beginRemoteAttachmentStop, settleRemoteAttachmentStop, markRemoteAttachmentStopUnknown, - findActiveRemoteAttachmentForPane + findActiveRemoteAttachmentForPane, + findLatestRemoteAttachmentForPane }) } diff --git a/src/main/runtime/orchestration/db/questions/question-threads.ts b/src/main/runtime/orchestration/db/questions/question-threads.ts index 54f85bc4e5f..84c8055ebe6 100644 --- a/src/main/runtime/orchestration/db/questions/question-threads.ts +++ b/src/main/runtime/orchestration/db/questions/question-threads.ts @@ -19,15 +19,12 @@ export function createQuestion( this.db.exec('BEGIN IMMEDIATE') try { this.requireRun(params.runId) + // A finished Dispatch still asks: its worker keeps talking to the coordinator after reporting. const dispatch = this.getDispatchContextById(params.dispatchId) - if ( - !dispatch || - dispatch.run_id !== params.runId || - (dispatch.status !== 'pending' && dispatch.status !== 'dispatched') - ) { + if (!dispatch || dispatch.run_id !== params.runId) { throw new OrchestrationError( 'dispatch_inactive', - `Dispatch ${params.dispatchId} is not active in Run ${params.runId}.` + `Dispatch ${params.dispatchId} is not in Run ${params.runId}.` ) } const message = this.insertMessage({ diff --git a/src/main/runtime/orchestration/worker-report-admission.ts b/src/main/runtime/orchestration/worker-report-admission.ts index 079e5d39c0d..82bb84aa31d 100644 --- a/src/main/runtime/orchestration/worker-report-admission.ts +++ b/src/main/runtime/orchestration/worker-report-admission.ts @@ -1,5 +1,6 @@ import { OrchestrationError } from './orchestration-error' -import type { WorkerDispatchState } from './types' +import { AGENT_PROMPT_STALLED_ERROR } from '../agent-prompt-submission-verification' +import type { DispatchContextRow, TaskRow, WorkerDispatchState } from './types' // Unknown start/stop cannot prove exit; the worker's report outranks uncertainty (docs/reference/ssh-execution-boundary.md). export const UNPROVEN_WORKER_STATES = [ @@ -52,3 +53,15 @@ export function assertWorkerCanReport(args: Parameters, + task: Pick +): boolean { + return ( + (dispatch.status === 'failed' || dispatch.status === 'circuit_broken') && + dispatch.last_failure === AGENT_PROMPT_STALLED_ERROR && + task.status === 'failed' + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts index 470325d1220..aa258b78ee7 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts @@ -13,6 +13,8 @@ import { import { resolveOrchestrationParty } from '../../../../orchestration/orchestration-party' import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence' import { assertWorkerCanReport } from '../../../../orchestration/worker-report-admission' +import { finishedWorkerDispatch, refuseFinishedRemoteWorker } from './finished-worker-mail' +import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup' import { askedQuestionReceipt, waitForQuestionAnswer @@ -68,25 +70,42 @@ export const ORCHESTRATION_ASK_METHODS = [ workerState: remoteAttachment.state }) } - const activeDispatch = db.getActiveDispatchForIdentity(from, paneKey) - if (!activeDispatch) { + const workingDispatch = db.getActiveDispatchForIdentity(from, paneKey) + // A finished worker still asks its coordinator; the question changes nothing about its Task. + const askingDispatch = + workingDispatch ?? + finishedWorkerDispatch( + db, + { + address: from, + paneKey: paneKey ?? null, + orcaSessionId: orchestrationCaller?.orcaSessionId ?? null, + processIncarnation: () => runtime.getTerminalProcessIncarnation(from) ?? undefined + }, + { run: undefined, dispatchId: undefined, dispatch: undefined }, + 'question' + ) + if (!askingDispatch) { + refuseFinishedRemoteWorker(db, paneKey, undefined) throw new OrchestrationError( 'dispatch_inactive', 'ask requires an active supervised Dispatch.' ) } - assertWorkerCanReport({ - dispatchId: activeDispatch.id, - from, - workerState: db.getWorkerDispatch(activeDispatch.id)?.state, - processCurrent: - !activeDispatch.process_incarnation || - db.isDispatchProcessCurrent({ - dispatchId: activeDispatch.id, - paneKey: paneKey ?? null, - processIncarnation: runtime.getTerminalProcessIncarnation(from) - }) - }) + if (workingDispatch) { + assertWorkerCanReport({ + dispatchId: workingDispatch.id, + from, + workerState: db.getWorkerDispatch(workingDispatch.id)?.state, + processCurrent: + !workingDispatch.process_incarnation || + db.isDispatchProcessCurrent({ + dispatchId: workingDispatch.id, + paneKey: paneKey ?? null, + processIncarnation: runtime.getTerminalProcessIncarnation(from) + }) + }) + } const options = params.options ?.split(',') @@ -94,27 +113,29 @@ export const ORCHESTRATION_ASK_METHODS = [ .filter(Boolean) ?? [] let question = params.resume ? db.getQuestion(params.resume) : undefined if (params.resume) { - if (!question || question.dispatch_id !== activeDispatch.id) { + // Its own question from a finished Dispatch stays resumable after a follow-up starts. + const askedBy = question && db.getDispatchContextById(question.dispatch_id) + if (!question || !askedBy || !isDispatchAssignee(askedBy, from, paneKey)) { throw new OrchestrationError( 'question_not_found', `Question ${params.resume} does not belong to this active Dispatch.` ) } } else { - const run = db.getRun(activeDispatch.run_id) + const run = db.getRun(askingDispatch.run_id) if (!run || run.legacy === 1) { throw new OrchestrationError( 'run_not_found', - `Run ${activeDispatch.run_id} was not found.` + `Run ${askingDispatch.run_id} was not found.` ) } if (params.run && params.run !== run.id) { throw new OrchestrationError( 'dispatch_run_mismatch', - `Dispatch ${activeDispatch.id} belongs to Run ${run.id}, not ${params.run}.` + `Dispatch ${askingDispatch.id} belongs to Run ${run.id}, not ${params.run}.` ) } - const reportTo = dispatchReportMailbox(db, activeDispatch) + const reportTo = dispatchReportMailbox(db, askingDispatch) if ( params.to && params.to !== `run:${run.id}` && @@ -124,14 +145,14 @@ export const ORCHESTRATION_ASK_METHODS = [ ) { throw new OrchestrationError( 'dispatch_run_mismatch', - `ask from Dispatch ${activeDispatch.id} must target its owning Run ${run.id}.` + `ask from Dispatch ${askingDispatch.id} must target its owning Run ${run.id}.` ) } // An explicit Run or coordinator target still reaches the Run; otherwise the asker's creator. const to = params.to && params.to !== reportTo ? `run:${run.id}` : reportTo const created = db.createQuestion({ runId: run.id, - dispatchId: activeDispatch.id, + dispatchId: askingDispatch.id, askerHandle: from, question: params.question as string, options, @@ -142,7 +163,8 @@ export const ORCHESTRATION_ASK_METHODS = [ } const questionId = question.message_id - if (!params.resume && params.wait === false) { + // A finished worker has no turn for an answer to wake, so it waits for it here. + if (!params.resume && params.wait === false && workingDispatch) { return askedQuestionReceipt(questionId, recordMutationReceipt) } return waitForQuestionAnswer({ diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.test.ts new file mode 100644 index 00000000000..b2bcc19bd34 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.test.ts @@ -0,0 +1,189 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { createOrchestrationRpcHarness } from '../rpc-test-harness' +import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture' + +describe('a worker after its worker_done', () => { + const harness = createOrchestrationRpcHarness() + afterEach(() => harness.cleanup()) + + it('still reaches its coordinator with every message, and none changes the finished Task', async () => { + const { db, runtime, ctx, activeRunId } = harness.setup() + const task = db.createTask({ spec: 'work' }) + const dispatch = createRootDispatch( + db, + task.id, + 'term_worker', + 'tab_worker:leaf_worker', + undefined, + 'runtime_test:term_worker:1' + ) + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_worker' + ? 'tab_worker:leaf_worker' + : handle === 'term_coord' + ? harness.coordinatorPaneKey + : null + ) + const send = (params: Record) => + harness.call('orchestration.send', { from: 'term_worker', ...params }, ctx) + const report = (outcome: string) => + JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome }) + + await send({ type: 'worker_done', subject: 'Done', payload: report('succeeded') }) + const settled = db.getTask(task.id) + + await send({ subject: 'No recipient' }) + await send({ to: `dispatch:${dispatch.id}`, subject: 'To my own Dispatch' }) + const again = await send({ + type: 'worker_done', + subject: 'Second report', + payload: report('failed') + }) + // It has no turn for a later answer to wake, so even a returning ask waits for the answer. + const asking = harness.call( + 'orchestration.ask', + { from: 'term_worker', question: 'Anything else?', wait: false, timeoutMs: 5_000 }, + ctx + ) + await vi.waitFor(() => expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(5)) + const question = db.getUnreadMessages(`run:${activeRunId}`)[4] + + expect(again).toMatchObject({ lifecycle: { action: 'settled', outcome: 'succeeded' } }) + expect(db.getTask(task.id)).toEqual(settled) + expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed') + expect( + db.getUnreadMessages(`run:${activeRunId}`).map((message) => [message.type, message.subject]) + ).toEqual([ + ['worker_done', 'Done'], + ['status', 'No recipient'], + ['status', 'To my own Dispatch'], + ['status', 'Second report'], + ['question', 'Question'] + ]) + await harness.call( + 'orchestration.reply', + { from: 'term_coord', id: question.id, body: 'Nothing else.' }, + ctx + ) + expect(await asking).toMatchObject({ answer: 'Nothing else.', timedOut: false }) + }) + + it("does not take a new process in the finished worker's pane for that worker", async () => { + const { db, runtime, ctx } = harness.setup() + const task = db.createTask({ spec: 'work' }) + const dispatch = createRootDispatch( + db, + task.id, + 'term_worker', + 'tab_worker:leaf_worker', + undefined, + 'runtime_test:term_worker:1' + ) + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_worker' ? 'tab_worker:leaf_worker' : null + ) + await harness.call( + 'orchestration.send', + { + from: 'term_worker', + type: 'worker_done', + subject: 'Done', + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) + }, + ctx + ) + vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime_test:term_worker:2') + + await expect( + harness.call('orchestration.send', { from: 'term_worker', subject: 'Not mine' }, ctx) + ).rejects.toMatchObject({ code: 'run_required' }) + }) + + it('resumes its own question after a follow-up starts on the same worker', async () => { + const { db, runtime, ctx } = harness.setup() + const task = db.createTask({ spec: 'work' }) + const dispatch = createRootDispatch(db, task.id, 'term_worker', 'tab_worker:leaf_worker') + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_worker' + ? 'tab_worker:leaf_worker' + : handle === 'term_coord' + ? harness.coordinatorPaneKey + : null + ) + await harness.call( + 'orchestration.send', + { + from: 'term_worker', + type: 'worker_done', + subject: 'Done', + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) + }, + ctx + ) + const asked = await harness.call( + 'orchestration.ask', + { from: 'term_worker', question: 'Anything else?', timeoutMs: 1 }, + ctx + ) + expect(asked).toMatchObject({ timedOut: true }) + const questionId = + asked && typeof asked === 'object' && 'messageId' in asked ? asked.messageId : '' + createRootDispatch( + db, + db.createTask({ spec: 'more' }).id, + 'term_worker', + 'tab_worker:leaf_worker' + ) + await harness.call( + 'orchestration.reply', + { from: 'term_coord', id: questionId, body: 'Nothing else.' }, + ctx + ) + + await expect( + harness.call('orchestration.ask', { from: 'term_worker', resume: questionId }, ctx) + ).resolves.toMatchObject({ answer: 'Nothing else.' }) + }) + + it('tells a finished worker on a paired host that its coordinator is out of reach', async () => { + const { db, runtime, ctx } = harness.setup(false) + db.createRemoteDispatchAttachment({ + dispatchId: 'ctx_remote', + runId: 'run_home', + taskId: 'task_remote', + homePeerFingerprint: 'peer_fp', + protocolVersion: 1, + runtimeEpoch: 'epoch_1', + mutationReceipt: { + callerFingerprint: 'peer_fp', + requestId: 'attach_1', + method: 'orchestration.federationAttachStart', + payloadHash: 'attach_payload' + } + }) + db.prepareRemoteAttachmentAuthority({ + dispatchId: 'ctx_remote', + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime_test:term_worker:1', + worktreeId: 'folder_workspace', + terminalHandle: 'term_worker', + setupState: 'not_applicable', + effects: [] + }) + db.markRemoteAttachmentReady('ctx_remote') + db.settleRemoteAttachmentInRelayTransaction('ctx_remote', 'succeeded') + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_worker' ? 'tab_worker:leaf_worker' : null + ) + + for (const [method, params] of [ + ['orchestration.send', { subject: 'Later' }], + ['orchestration.send', { to: 'dispatch:ctx_remote', subject: 'To my own Dispatch' }], + ['orchestration.ask', { question: 'Anything else?', wait: false }] + ] as const) { + await expect( + harness.call(method, { from: 'term_worker', ...params }, ctx) + ).rejects.toMatchObject({ code: 'run_home_unreachable' }) + } + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts new file mode 100644 index 00000000000..35169158ecf --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts @@ -0,0 +1,101 @@ +import type { OrchestrationDb } from '../../../../orchestration/db' +import type { OrchestrationCallerIdentity } from '../../../../orchestration/orchestration-caller-identity' +import type { DispatchContextRow } from '../../../../orchestration/types' +import { dispatchReportMailbox } from '../../../../orchestration/dispatch-report-mailbox' +import { reportCorrectsUnobservedPrompt } from '../../../../orchestration/worker-report-admission' +import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup' +import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import { isDispatchMutationMessageType } from '../schemas' +import type { MessageRouting } from '../routing' + +/** A terminal also proves its process, so a new process in the same pane is not taken for the old worker; + * a session caller is already bound to its own identity. */ +type FinishedSender = Pick & { + processIncarnation: () => string | undefined +} + +/** The Dispatch a worker finished and is still talking about: the one its message names, or, naming none + * and coordinating nothing, the last one it was given. What it sends there reaches that Dispatch's coordinator. */ +export function finishedWorkerDispatch( + db: OrchestrationDb, + sender: FinishedSender, + routing: Pick, + type: string | undefined +): DispatchContextRow | undefined { + // A late heartbeat is already dropped quietly; it says nothing the coordinator needs. + if (type === 'heartbeat' || (!routing.dispatchId && routing.run)) { + return undefined + } + const dispatch = routing.dispatchId + ? routing.dispatch + : db.getLatestDispatchForTerminal(sender.address, sender.paneKey ?? undefined) + if ( + !dispatch || + dispatch.status === 'pending' || + dispatch.status === 'dispatched' || + !isDispatchAssignee(dispatch, sender.address, sender.paneKey) || + (!sender.orcaSessionId && + dispatch.process_incarnation && + !db.isDispatchProcessCurrent({ + dispatchId: dispatch.id, + paneKey: sender.paneKey, + processIncarnation: sender.processIncarnation() ?? null + })) || + db.getFederatedDispatch(dispatch.id) + ) { + return undefined + } + const task = type === 'worker_done' ? db.getTask(dispatch.task_id) : undefined + return task && reportCorrectsUnobservedPrompt(dispatch, task) ? undefined : dispatch +} + +/** Mail from a finished worker is ordinary mail to its coordinator, so no report or escalation reopens the Task. */ +export function finishedWorkerSend

( + db: OrchestrationDb, + sender: FinishedSender, + routing: MessageRouting, + params: P +): + | { params: P; routing: MessageRouting; receipt: (receipt: T) => T } + | undefined { + const finished = finishedWorkerDispatch(db, sender, routing, params.type) + if (!finished) { + return undefined + } + const ownDispatch = !params.to || params.to === `dispatch:${finished.id}` + return { + params: { + ...params, + type: isDispatchMutationMessageType(params.type) ? 'status' : params.type, + to: ownDispatch ? dispatchReportMailbox(db, finished) : params.to + }, + routing: { run: db.getRun(finished.run_id), dispatchId: undefined, dispatch: undefined }, + // Older CLIs confirm a worker_done only by a verdict; the Task's recorded one stands. + receipt: (receipt) => + params.type === 'worker_done' + ? { + ...receipt, + lifecycle: { + action: 'settled', + outcome: finished.status === 'completed' ? 'succeeded' : 'failed' + }, + next: 'Delivered to your coordinator as a message; your finished task is unchanged. End your turn.' + } + : receipt + } +} + +/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. */ +export function refuseFinishedRemoteWorker( + db: OrchestrationDb, + paneKey: string | undefined, + to: string | undefined +): void { + const attachment = paneKey ? db.findLatestRemoteAttachmentForPane(paneKey) : undefined + if (attachment && (!to || to === `dispatch:${attachment.dispatch_id}`)) { + throw new OrchestrationError( + 'run_home_unreachable', + `Dispatch ${attachment.dispatch_id} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.` + ) + } +} diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts index 0ba04050433..f9acbd46663 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts @@ -34,6 +34,7 @@ import { sendFederatedControlMail } from './send-control-mail' import { orchestrationCallerIdentity } from '../runs/run-scope' import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence' import { planWorkerFollowUp, startWorkerFollowUp } from './worker-follow-up' +import { finishedWorkerSend, refuseFinishedRemoteWorker } from './finished-worker-mail' import { followThroughWorkerReport, passHeldWorkerMail, @@ -103,6 +104,10 @@ export const ORCHESTRATION_SEND_METHODS = [ evidence: orchestrationCompatibilityEvidence }) } + const processIncarnation = () => + attestedCaller?.processIncarnation ?? + runtime.getTerminalProcessIncarnation(from) ?? + undefined const remoteAttachment = senderPaneKey ? db.findActiveRemoteAttachmentForPane(senderPaneKey) : undefined @@ -114,24 +119,26 @@ export const ORCHESTRATION_SEND_METHODS = [ from, senderPaneKey, remoteAttachment, - processIncarnation: - attestedCaller?.processIncarnation ?? - runtime.getTerminalProcessIncarnation(from) ?? - undefined, + processIncarnation: processIncarnation(), signal }) } + refuseFinishedRemoteWorker(db, senderPaneKey, params.to) const runGroup = params.to && isGroupAddress(params.to) && !params.to.toLowerCase().startsWith('@worktree:') // Run groups validate their own audience; message scope cannot select a parent Dispatch. - const routing = resolveMessageRun(runtime, { + let routing = resolveMessageRun(runtime, { sender, to: params.to, runId: runGroup ? undefined : params.run, payload: runGroup ? undefined : params.payload, type: params.type }) + const finished = runGroup + ? undefined + : finishedWorkerSend(db, { ...sender, processIncarnation }, routing, params) + ;({ params, routing } = finished ?? { params, routing }) if ( params.type === 'worker_done' && !isWorkerReportOutcome(parseRemoteWorkerPayload(params.payload).outcome) @@ -271,14 +278,12 @@ export const ORCHESTRATION_SEND_METHODS = [ messageRunId, senderPaneKey, legacyCoordinatorRunId, - resolveProcessIncarnation: () => - attestedCaller?.processIncarnation ?? - runtime.getTerminalProcessIncarnation(from) ?? - undefined, + resolveProcessIncarnation: processIncarnation, revalidateLegacyCoordinator, recordMutationReceipt, markWorkerDoneMutationEffectFree, - withSendWarnings + withSendWarnings: (sent) => + finished?.receipt(withSendWarnings(sent)) ?? withSendWarnings(sent) }) followThroughWorkerReport( runtime, diff --git a/src/main/runtime/rpc/methods/orchestration/routing.ts b/src/main/runtime/rpc/methods/orchestration/routing.ts index 71cf7cb873d..4dece74a68d 100644 --- a/src/main/runtime/rpc/methods/orchestration/routing.ts +++ b/src/main/runtime/rpc/methods/orchestration/routing.ts @@ -1,5 +1,5 @@ import type { MessageType } from '../../../orchestration/db' -import type { RunRow } from '../../../orchestration/types' +import type { DispatchContextRow, RunRow } from '../../../orchestration/types' import type { OrcaRuntimeService } from '../../../orca-runtime' import type { OrchestrationCallerIdentity } from '../../../orchestration/orchestration-caller-identity' import { MESSAGE_TYPES } from '../../../orchestration/types' @@ -20,6 +20,12 @@ export function parseMessageTypes(rawTypes: string | undefined): MessageType[] | return types && types.length > 0 ? types : undefined } +export type MessageRouting = { + run: RunRow | undefined + dispatchId: string | undefined + dispatch: DispatchContextRow | undefined +} + export function resolveMessageRun( runtime: OrcaRuntimeService, params: { @@ -30,7 +36,7 @@ export function resolveMessageRun( payload?: string type?: string } -): { run: RunRow | undefined; dispatchId: string | undefined } { +): MessageRouting { const db = runtime.getOrchestrationDb() let dispatchId: string | undefined if (params.payload) { @@ -111,7 +117,7 @@ export function resolveMessageRun( ) } } - return { run, dispatchId: dispatch?.id ?? dispatchId } + return { run, dispatchId: dispatch?.id ?? dispatchId, dispatch } } export function legacyWorkerDeliveryContract(