Let a finished worker keep messaging its coordinator (#38357)

* Let a finished worker keep messaging its coordinator

* Tell a finished remote worker why its coordinator is out of reach

* Let a finished worker's ask wait for its answer instead of starting a follow-up

* Share the assignee check, keep the finished lookup in dispatch-lookup, and fix review findings

* Match a finished terminal worker by its process too; cover resume after a follow-up
This commit is contained in:
Jinwoo Hong
2026-11-05 02:42:50 -05:00
committed by GitHub
parent a5b1b72994
commit 0d696aef33
11 changed files with 456 additions and 101 deletions
+2 -2
View File
@@ -79,7 +79,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
'On Windows PowerShell, quote group addresses such as --to "@all" or --to "@worktree:<id>".',
"worker_done and heartbeat are exact-Dispatch signals and cannot target groups; omit --to to use the Dispatch's Run mailbox.",
'worker_done requires --outcome succeeded or --outcome failed.',
'From an active Dispatch, an omitted recipient defaults to its owning Run mailbox.',
'From a worker, an omitted recipient defaults to its coordinator, also after it finished; mail sent after finishing, worker_done included, leaves the finished task unchanged.',
'Use --to dispatch:<id> for attempt-specific coordinator guidance; Orca durably relays it to a connected worker server.',
'A worker_done with the active task/dispatch IDs completes that task only from the dispatched pane. When stable pane identity is unavailable, the sender handle must exactly match the dispatch assignee; injected preambles include the correct --from value.',
'Prefer --task-id/--dispatch-id/etc. over raw --payload JSON in worker commands; PowerShell strips JSON quotes easily.'
@@ -239,7 +239,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
],
identityFlagRoles: { from: 'caller' },
notes: [
'From an active Dispatch, a new question goes to its owning Run mailbox and ask returns at once: end your turn, and Orca wakes you with the answer as mail. --wait and --resume block until the answer instead; --timeout-ms accepts 1 to 1800000 ms per wait (default 600000 = 10 minutes, max 30 minutes), and a timeout leaves the question pending.'
'From a worker, a new question goes to its coordinator and ask returns at once: end your turn, and Orca wakes you with the answer as mail. A worker that already finished waits for the answer instead. --wait and --resume block until the answer instead; --timeout-ms accepts 1 to 1800000 ms per wait (default 600000 = 10 minutes, max 30 minutes), and a timeout leaves the question pending.'
]
},
{
@@ -13,21 +13,22 @@ import {
equivalentDispatchMailboxOwners
} from '../dispatch-mailbox-selection'
// Hoist explicit projections so graph-publish queries hit the statement cache.
const ACTIVE_DISPATCH_BY_HANDLE_SQL =
// Newest-first avoids selecting a stale row when one handle has multiple active Dispatches.
// Hoist explicit projections so graph-publish queries hit the statement cache; newest first, since
// one handle can hold several active Dispatches.
const assigneeSql = (where: string, status: string) =>
`SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts
WHERE assignee_handle = ? AND status IN ('pending', 'dispatched')
ORDER BY rowid DESC LIMIT 1`
const ACTIVE_DISPATCH_BY_PANE_KEY_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts
WHERE assignee_pane_key = ? AND status IN ('pending', 'dispatched')
ORDER BY rowid DESC LIMIT 1`
const ACTIVE_DISPATCH_BY_PANE_SUFFIX_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts
WHERE assignee_pane_key IS NOT NULL
AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1
AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?
ORDER BY rowid DESC LIMIT 1`
const LATEST_DISPATCH_BY_HANDLE_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1`
WHERE ${where} ${status} ORDER BY rowid DESC LIMIT 1`
const BY_PANE_SUFFIX = `assignee_pane_key IS NOT NULL AND instr(assignee_pane_key, ':') > 1
AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?`
const assigneeLookups = (status: string) => ({
byHandle: assigneeSql('assignee_handle = ?', status),
byPaneKey: assigneeSql('assignee_pane_key = ?', status),
byPaneSuffix: assigneeSql(BY_PANE_SUFFIX, status)
})
const ASSIGNEE_SQL = {
active: assigneeLookups("AND status IN ('pending', 'dispatched')"),
latest: assigneeLookups('')
}
function presentationAddressKey(column: string, address: string | null): string {
return `${column}:${address}`
@@ -232,16 +233,23 @@ export function isDispatchMessageSender(
if (params.allowCanonicalDispatchHandle && params.handle === `dispatch:${dispatch.id}`) {
return true
}
return isDispatchAssignee(dispatch, params.handle, params.paneKey)
}
/** Whether this handle or pane was given the Dispatch, whatever its status. */
export function isDispatchAssignee(
dispatch: Pick<DispatchContextRow, 'assignee_handle' | 'assignee_pane_key'>,
handle: string,
paneKey: string | null | undefined
): boolean {
if (
params.paneKey &&
paneKey &&
dispatch.assignee_pane_key &&
isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey)
isEquivalentPaneKey(dispatch.assignee_pane_key, paneKey)
) {
return true
}
return (
params.handle === dispatch.assignee_handle && (!params.paneKey || !dispatch.assignee_pane_key)
)
return handle === dispatch.assignee_handle && (!paneKey || !dispatch.assignee_pane_key)
}
export function findActiveDispatchForAssignee(
@@ -249,38 +257,37 @@ export function findActiveDispatchForAssignee(
assigneeHandle: string,
assigneePaneKey?: string
): DispatchContextRow | undefined {
const byHandle = this.db.prepare(ACTIVE_DISPATCH_BY_HANDLE_SQL).get(assigneeHandle) as
| DispatchContextRow
| undefined
if (byHandle) {
return byHandle
}
if (!assigneePaneKey) {
return undefined
}
const exactPane = this.db.prepare(ACTIVE_DISPATCH_BY_PANE_KEY_SQL).get(assigneePaneKey) as
| DispatchContextRow
| undefined
if (exactPane) {
return exactPane
}
if (!parsePaneKey(assigneePaneKey)) {
return undefined
}
return this.db
.prepare(ACTIVE_DISPATCH_BY_PANE_SUFFIX_SQL)
.get(paneKeyMatchSuffix(assigneePaneKey)) as DispatchContextRow | undefined
return findDispatchForAssignee(this, 'active', assigneeHandle, assigneePaneKey)
}
/** The newest Dispatch, settled or not; with a pane, also after the handle was reminted. */
export function getLatestDispatchForTerminal(
this: OrchestrationDb,
handle: string
handle: string,
paneKey?: string
): DispatchContextRow | undefined {
return this.db.prepare(LATEST_DISPATCH_BY_HANDLE_SQL).get(handle) as
| DispatchContextRow
| undefined
return findDispatchForAssignee(this, 'latest', handle, paneKey)
}
function findDispatchForAssignee(
db: OrchestrationDb,
kind: keyof typeof ASSIGNEE_SQL,
handle: string,
paneKey: string | undefined
): DispatchContextRow | undefined {
const sql = ASSIGNEE_SQL[kind]
const read = (statement: string, value: string) =>
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: each statement selects the schema-pinned Dispatch projection.
db.db.prepare(statement).get(value) as DispatchContextRow | undefined
const byHandle = read(sql.byHandle, handle)
if (byHandle || !paneKey) {
return byHandle
}
const exactPane = read(sql.byPaneKey, paneKey)
if (exactPane || !parsePaneKey(paneKey)) {
return exactPane
}
return read(sql.byPaneSuffix, paneKeyMatchSuffix(paneKey))
}
/** Helpers this Dispatch started that have not settled; their reports wake their creator. */
@@ -1,13 +1,13 @@
import { markUnreadFollowUpMail } from './worker-follow-up-lineage'
import type { WorkerReportOutcome, WorkerReportSettlement } from '../../types'
import type { OrchestrationDb } from '../orchestration-db'
import { AGENT_PROMPT_STALLED_ERROR } from '../../../agent-prompt-submission-verification'
import { settleActiveDispatchesForTask } from './dispatch-completion'
import { getActiveDispatchForTask } from './task-dispatch-reconciliation'
import { transitionLifecycleWithDb } from '../lifecycle-transition'
import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner'
import {
isWorkerStateIn,
reportCorrectsUnobservedPrompt,
SETTLEABLE_WORKER_STATES,
UNPROVEN_WORKER_STATES
} from '../../worker-report-admission'
@@ -93,11 +93,8 @@ export function settleWorkerReportInTransaction(
const expectedDispatchStatus = params.outcome === 'succeeded' ? 'completed' : 'failed'
const expectedTaskStatus = params.outcome === 'succeeded' ? 'completed' : 'failed'
// A first-hand worker report can correct an unobserved prompt failure, even after circuit break; check before duplicate detection (#16095).
const settledByUnobservedPrompt =
(dispatch.status === 'failed' || dispatch.status === 'circuit_broken') &&
dispatch.last_failure === AGENT_PROMPT_STALLED_ERROR &&
task.status === 'failed'
// Check before duplicate detection (#16095).
const settledByUnobservedPrompt = reportCorrectsUnobservedPrompt(dispatch, task)
const reportingWorker = this.getWorkerDispatch(params.dispatchId)
const settledDispatchStatus = dispatch.status === 'circuit_broken' ? 'failed' : dispatch.status
if (
@@ -73,22 +73,38 @@ export function markRemoteAttachmentStopUnknown(
export function findActiveRemoteAttachmentForPane(
this: OrchestrationDb,
paneKey: string
): RemoteDispatchAttachmentRow | undefined {
return findRemoteAttachmentForPane(this, paneKey, `${potentiallyLiveRemoteAttachmentSql()} AND`)
}
/** The newest remote Dispatch this pane ran, finished or not. */
export function findLatestRemoteAttachmentForPane(
this: OrchestrationDb,
paneKey: string
): RemoteDispatchAttachmentRow | undefined {
return findRemoteAttachmentForPane(this, paneKey, '')
}
function findRemoteAttachmentForPane(
db: OrchestrationDb,
paneKey: string,
stateFilter: string
): RemoteDispatchAttachmentRow | undefined {
if (!parsePaneKey(paneKey)) {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The schema-pinned attachment projection returns RemoteDispatchAttachmentRow rows; the adapter exposes unknown.
return this.db
return db.db
.prepare(
`SELECT ${REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST} FROM remote_dispatch_attachments
WHERE ${potentiallyLiveRemoteAttachmentSql()} AND pane_key = ?
WHERE ${stateFilter} pane_key = ?
ORDER BY rowid DESC LIMIT 1`
)
.get(paneKey) as RemoteDispatchAttachmentRow | undefined
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The schema-pinned attachment projection returns RemoteDispatchAttachmentRow rows; the adapter exposes unknown.
return this.db
return db.db
.prepare(
`SELECT ${REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST} FROM remote_dispatch_attachments
WHERE ${potentiallyLiveRemoteAttachmentSql()} AND pane_key IS NOT NULL
WHERE ${stateFilter} pane_key IS NOT NULL
AND instr(pane_key, ':') > 1
AND ${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL} = ?
ORDER BY rowid DESC LIMIT 1`
@@ -101,6 +117,7 @@ export type RemoteDispatchAttachmentStopMethods = {
settleRemoteAttachmentStop: typeof settleRemoteAttachmentStop
markRemoteAttachmentStopUnknown: typeof markRemoteAttachmentStopUnknown
findActiveRemoteAttachmentForPane: typeof findActiveRemoteAttachmentForPane
findLatestRemoteAttachmentForPane: typeof findLatestRemoteAttachmentForPane
}
export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }): void {
@@ -108,6 +125,7 @@ export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }):
beginRemoteAttachmentStop,
settleRemoteAttachmentStop,
markRemoteAttachmentStopUnknown,
findActiveRemoteAttachmentForPane
findActiveRemoteAttachmentForPane,
findLatestRemoteAttachmentForPane
})
}
@@ -19,15 +19,12 @@ export function createQuestion(
this.db.exec('BEGIN IMMEDIATE')
try {
this.requireRun(params.runId)
// A finished Dispatch still asks: its worker keeps talking to the coordinator after reporting.
const dispatch = this.getDispatchContextById(params.dispatchId)
if (
!dispatch ||
dispatch.run_id !== params.runId ||
(dispatch.status !== 'pending' && dispatch.status !== 'dispatched')
) {
if (!dispatch || dispatch.run_id !== params.runId) {
throw new OrchestrationError(
'dispatch_inactive',
`Dispatch ${params.dispatchId} is not active in Run ${params.runId}.`
`Dispatch ${params.dispatchId} is not in Run ${params.runId}.`
)
}
const message = this.insertMessage({
@@ -1,5 +1,6 @@
import { OrchestrationError } from './orchestration-error'
import type { WorkerDispatchState } from './types'
import { AGENT_PROMPT_STALLED_ERROR } from '../agent-prompt-submission-verification'
import type { DispatchContextRow, TaskRow, WorkerDispatchState } from './types'
// Unknown start/stop cannot prove exit; the worker's report outranks uncertainty (docs/reference/ssh-execution-boundary.md).
export const UNPROVEN_WORKER_STATES = [
@@ -52,3 +53,15 @@ export function assertWorkerCanReport(args: Parameters<typeof workerReportRefusa
throw new OrchestrationError(refusal.code, refusal.reason)
}
}
/** A first-hand worker report can correct an unobserved prompt failure, even after circuit break. */
export function reportCorrectsUnobservedPrompt(
dispatch: Pick<DispatchContextRow, 'status' | 'last_failure'>,
task: Pick<TaskRow, 'status'>
): boolean {
return (
(dispatch.status === 'failed' || dispatch.status === 'circuit_broken') &&
dispatch.last_failure === AGENT_PROMPT_STALLED_ERROR &&
task.status === 'failed'
)
}
@@ -13,6 +13,8 @@ import {
import { resolveOrchestrationParty } from '../../../../orchestration/orchestration-party'
import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence'
import { assertWorkerCanReport } from '../../../../orchestration/worker-report-admission'
import { finishedWorkerDispatch, refuseFinishedRemoteWorker } from './finished-worker-mail'
import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup'
import {
askedQuestionReceipt,
waitForQuestionAnswer
@@ -68,25 +70,42 @@ export const ORCHESTRATION_ASK_METHODS = [
workerState: remoteAttachment.state
})
}
const activeDispatch = db.getActiveDispatchForIdentity(from, paneKey)
if (!activeDispatch) {
const workingDispatch = db.getActiveDispatchForIdentity(from, paneKey)
// A finished worker still asks its coordinator; the question changes nothing about its Task.
const askingDispatch =
workingDispatch ??
finishedWorkerDispatch(
db,
{
address: from,
paneKey: paneKey ?? null,
orcaSessionId: orchestrationCaller?.orcaSessionId ?? null,
processIncarnation: () => runtime.getTerminalProcessIncarnation(from) ?? undefined
},
{ run: undefined, dispatchId: undefined, dispatch: undefined },
'question'
)
if (!askingDispatch) {
refuseFinishedRemoteWorker(db, paneKey, undefined)
throw new OrchestrationError(
'dispatch_inactive',
'ask requires an active supervised Dispatch.'
)
}
assertWorkerCanReport({
dispatchId: activeDispatch.id,
from,
workerState: db.getWorkerDispatch(activeDispatch.id)?.state,
processCurrent:
!activeDispatch.process_incarnation ||
db.isDispatchProcessCurrent({
dispatchId: activeDispatch.id,
paneKey: paneKey ?? null,
processIncarnation: runtime.getTerminalProcessIncarnation(from)
})
})
if (workingDispatch) {
assertWorkerCanReport({
dispatchId: workingDispatch.id,
from,
workerState: db.getWorkerDispatch(workingDispatch.id)?.state,
processCurrent:
!workingDispatch.process_incarnation ||
db.isDispatchProcessCurrent({
dispatchId: workingDispatch.id,
paneKey: paneKey ?? null,
processIncarnation: runtime.getTerminalProcessIncarnation(from)
})
})
}
const options =
params.options
?.split(',')
@@ -94,27 +113,29 @@ export const ORCHESTRATION_ASK_METHODS = [
.filter(Boolean) ?? []
let question = params.resume ? db.getQuestion(params.resume) : undefined
if (params.resume) {
if (!question || question.dispatch_id !== activeDispatch.id) {
// Its own question from a finished Dispatch stays resumable after a follow-up starts.
const askedBy = question && db.getDispatchContextById(question.dispatch_id)
if (!question || !askedBy || !isDispatchAssignee(askedBy, from, paneKey)) {
throw new OrchestrationError(
'question_not_found',
`Question ${params.resume} does not belong to this active Dispatch.`
)
}
} else {
const run = db.getRun(activeDispatch.run_id)
const run = db.getRun(askingDispatch.run_id)
if (!run || run.legacy === 1) {
throw new OrchestrationError(
'run_not_found',
`Run ${activeDispatch.run_id} was not found.`
`Run ${askingDispatch.run_id} was not found.`
)
}
if (params.run && params.run !== run.id) {
throw new OrchestrationError(
'dispatch_run_mismatch',
`Dispatch ${activeDispatch.id} belongs to Run ${run.id}, not ${params.run}.`
`Dispatch ${askingDispatch.id} belongs to Run ${run.id}, not ${params.run}.`
)
}
const reportTo = dispatchReportMailbox(db, activeDispatch)
const reportTo = dispatchReportMailbox(db, askingDispatch)
if (
params.to &&
params.to !== `run:${run.id}` &&
@@ -124,14 +145,14 @@ export const ORCHESTRATION_ASK_METHODS = [
) {
throw new OrchestrationError(
'dispatch_run_mismatch',
`ask from Dispatch ${activeDispatch.id} must target its owning Run ${run.id}.`
`ask from Dispatch ${askingDispatch.id} must target its owning Run ${run.id}.`
)
}
// An explicit Run or coordinator target still reaches the Run; otherwise the asker's creator.
const to = params.to && params.to !== reportTo ? `run:${run.id}` : reportTo
const created = db.createQuestion({
runId: run.id,
dispatchId: activeDispatch.id,
dispatchId: askingDispatch.id,
askerHandle: from,
question: params.question as string,
options,
@@ -142,7 +163,8 @@ export const ORCHESTRATION_ASK_METHODS = [
}
const questionId = question.message_id
if (!params.resume && params.wait === false) {
// A finished worker has no turn for an answer to wake, so it waits for it here.
if (!params.resume && params.wait === false && workingDispatch) {
return askedQuestionReceipt(questionId, recordMutationReceipt)
}
return waitForQuestionAnswer({
@@ -0,0 +1,189 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { createOrchestrationRpcHarness } from '../rpc-test-harness'
import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture'
describe('a worker after its worker_done', () => {
const harness = createOrchestrationRpcHarness()
afterEach(() => harness.cleanup())
it('still reaches its coordinator with every message, and none changes the finished Task', async () => {
const { db, runtime, ctx, activeRunId } = harness.setup()
const task = db.createTask({ spec: 'work' })
const dispatch = createRootDispatch(
db,
task.id,
'term_worker',
'tab_worker:leaf_worker',
undefined,
'runtime_test:term_worker:1'
)
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
handle === 'term_worker'
? 'tab_worker:leaf_worker'
: handle === 'term_coord'
? harness.coordinatorPaneKey
: null
)
const send = (params: Record<string, unknown>) =>
harness.call('orchestration.send', { from: 'term_worker', ...params }, ctx)
const report = (outcome: string) =>
JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome })
await send({ type: 'worker_done', subject: 'Done', payload: report('succeeded') })
const settled = db.getTask(task.id)
await send({ subject: 'No recipient' })
await send({ to: `dispatch:${dispatch.id}`, subject: 'To my own Dispatch' })
const again = await send({
type: 'worker_done',
subject: 'Second report',
payload: report('failed')
})
// It has no turn for a later answer to wake, so even a returning ask waits for the answer.
const asking = harness.call(
'orchestration.ask',
{ from: 'term_worker', question: 'Anything else?', wait: false, timeoutMs: 5_000 },
ctx
)
await vi.waitFor(() => expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(5))
const question = db.getUnreadMessages(`run:${activeRunId}`)[4]
expect(again).toMatchObject({ lifecycle: { action: 'settled', outcome: 'succeeded' } })
expect(db.getTask(task.id)).toEqual(settled)
expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed')
expect(
db.getUnreadMessages(`run:${activeRunId}`).map((message) => [message.type, message.subject])
).toEqual([
['worker_done', 'Done'],
['status', 'No recipient'],
['status', 'To my own Dispatch'],
['status', 'Second report'],
['question', 'Question']
])
await harness.call(
'orchestration.reply',
{ from: 'term_coord', id: question.id, body: 'Nothing else.' },
ctx
)
expect(await asking).toMatchObject({ answer: 'Nothing else.', timedOut: false })
})
it("does not take a new process in the finished worker's pane for that worker", async () => {
const { db, runtime, ctx } = harness.setup()
const task = db.createTask({ spec: 'work' })
const dispatch = createRootDispatch(
db,
task.id,
'term_worker',
'tab_worker:leaf_worker',
undefined,
'runtime_test:term_worker:1'
)
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
handle === 'term_worker' ? 'tab_worker:leaf_worker' : null
)
await harness.call(
'orchestration.send',
{
from: 'term_worker',
type: 'worker_done',
subject: 'Done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' })
},
ctx
)
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime_test:term_worker:2')
await expect(
harness.call('orchestration.send', { from: 'term_worker', subject: 'Not mine' }, ctx)
).rejects.toMatchObject({ code: 'run_required' })
})
it('resumes its own question after a follow-up starts on the same worker', async () => {
const { db, runtime, ctx } = harness.setup()
const task = db.createTask({ spec: 'work' })
const dispatch = createRootDispatch(db, task.id, 'term_worker', 'tab_worker:leaf_worker')
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
handle === 'term_worker'
? 'tab_worker:leaf_worker'
: handle === 'term_coord'
? harness.coordinatorPaneKey
: null
)
await harness.call(
'orchestration.send',
{
from: 'term_worker',
type: 'worker_done',
subject: 'Done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' })
},
ctx
)
const asked = await harness.call(
'orchestration.ask',
{ from: 'term_worker', question: 'Anything else?', timeoutMs: 1 },
ctx
)
expect(asked).toMatchObject({ timedOut: true })
const questionId =
asked && typeof asked === 'object' && 'messageId' in asked ? asked.messageId : ''
createRootDispatch(
db,
db.createTask({ spec: 'more' }).id,
'term_worker',
'tab_worker:leaf_worker'
)
await harness.call(
'orchestration.reply',
{ from: 'term_coord', id: questionId, body: 'Nothing else.' },
ctx
)
await expect(
harness.call('orchestration.ask', { from: 'term_worker', resume: questionId }, ctx)
).resolves.toMatchObject({ answer: 'Nothing else.' })
})
it('tells a finished worker on a paired host that its coordinator is out of reach', async () => {
const { db, runtime, ctx } = harness.setup(false)
db.createRemoteDispatchAttachment({
dispatchId: 'ctx_remote',
runId: 'run_home',
taskId: 'task_remote',
homePeerFingerprint: 'peer_fp',
protocolVersion: 1,
runtimeEpoch: 'epoch_1',
mutationReceipt: {
callerFingerprint: 'peer_fp',
requestId: 'attach_1',
method: 'orchestration.federationAttachStart',
payloadHash: 'attach_payload'
}
})
db.prepareRemoteAttachmentAuthority({
dispatchId: 'ctx_remote',
paneKey: 'tab_worker:leaf_worker',
processIncarnation: 'runtime_test:term_worker:1',
worktreeId: 'folder_workspace',
terminalHandle: 'term_worker',
setupState: 'not_applicable',
effects: []
})
db.markRemoteAttachmentReady('ctx_remote')
db.settleRemoteAttachmentInRelayTransaction('ctx_remote', 'succeeded')
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
handle === 'term_worker' ? 'tab_worker:leaf_worker' : null
)
for (const [method, params] of [
['orchestration.send', { subject: 'Later' }],
['orchestration.send', { to: 'dispatch:ctx_remote', subject: 'To my own Dispatch' }],
['orchestration.ask', { question: 'Anything else?', wait: false }]
] as const) {
await expect(
harness.call(method, { from: 'term_worker', ...params }, ctx)
).rejects.toMatchObject({ code: 'run_home_unreachable' })
}
})
})
@@ -0,0 +1,101 @@
import type { OrchestrationDb } from '../../../../orchestration/db'
import type { OrchestrationCallerIdentity } from '../../../../orchestration/orchestration-caller-identity'
import type { DispatchContextRow } from '../../../../orchestration/types'
import { dispatchReportMailbox } from '../../../../orchestration/dispatch-report-mailbox'
import { reportCorrectsUnobservedPrompt } from '../../../../orchestration/worker-report-admission'
import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { isDispatchMutationMessageType } from '../schemas'
import type { MessageRouting } from '../routing'
/** A terminal also proves its process, so a new process in the same pane is not taken for the old worker;
* a session caller is already bound to its own identity. */
type FinishedSender = Pick<OrchestrationCallerIdentity, 'address' | 'paneKey' | 'orcaSessionId'> & {
processIncarnation: () => string | undefined
}
/** The Dispatch a worker finished and is still talking about: the one its message names, or, naming none
* and coordinating nothing, the last one it was given. What it sends there reaches that Dispatch's coordinator. */
export function finishedWorkerDispatch(
db: OrchestrationDb,
sender: FinishedSender,
routing: Pick<MessageRouting, 'run' | 'dispatchId' | 'dispatch'>,
type: string | undefined
): DispatchContextRow | undefined {
// A late heartbeat is already dropped quietly; it says nothing the coordinator needs.
if (type === 'heartbeat' || (!routing.dispatchId && routing.run)) {
return undefined
}
const dispatch = routing.dispatchId
? routing.dispatch
: db.getLatestDispatchForTerminal(sender.address, sender.paneKey ?? undefined)
if (
!dispatch ||
dispatch.status === 'pending' ||
dispatch.status === 'dispatched' ||
!isDispatchAssignee(dispatch, sender.address, sender.paneKey) ||
(!sender.orcaSessionId &&
dispatch.process_incarnation &&
!db.isDispatchProcessCurrent({
dispatchId: dispatch.id,
paneKey: sender.paneKey,
processIncarnation: sender.processIncarnation() ?? null
})) ||
db.getFederatedDispatch(dispatch.id)
) {
return undefined
}
const task = type === 'worker_done' ? db.getTask(dispatch.task_id) : undefined
return task && reportCorrectsUnobservedPrompt(dispatch, task) ? undefined : dispatch
}
/** Mail from a finished worker is ordinary mail to its coordinator, so no report or escalation reopens the Task. */
export function finishedWorkerSend<P extends { to?: string; type?: string }>(
db: OrchestrationDb,
sender: FinishedSender,
routing: MessageRouting,
params: P
):
| { params: P; routing: MessageRouting; receipt: <T extends object>(receipt: T) => T }
| undefined {
const finished = finishedWorkerDispatch(db, sender, routing, params.type)
if (!finished) {
return undefined
}
const ownDispatch = !params.to || params.to === `dispatch:${finished.id}`
return {
params: {
...params,
type: isDispatchMutationMessageType(params.type) ? 'status' : params.type,
to: ownDispatch ? dispatchReportMailbox(db, finished) : params.to
},
routing: { run: db.getRun(finished.run_id), dispatchId: undefined, dispatch: undefined },
// Older CLIs confirm a worker_done only by a verdict; the Task's recorded one stands.
receipt: (receipt) =>
params.type === 'worker_done'
? {
...receipt,
lifecycle: {
action: 'settled',
outcome: finished.status === 'completed' ? 'succeeded' : 'failed'
},
next: 'Delivered to your coordinator as a message; your finished task is unchanged. End your turn.'
}
: receipt
}
}
/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. */
export function refuseFinishedRemoteWorker(
db: OrchestrationDb,
paneKey: string | undefined,
to: string | undefined
): void {
const attachment = paneKey ? db.findLatestRemoteAttachmentForPane(paneKey) : undefined
if (attachment && (!to || to === `dispatch:${attachment.dispatch_id}`)) {
throw new OrchestrationError(
'run_home_unreachable',
`Dispatch ${attachment.dispatch_id} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.`
)
}
}
@@ -34,6 +34,7 @@ import { sendFederatedControlMail } from './send-control-mail'
import { orchestrationCallerIdentity } from '../runs/run-scope'
import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence'
import { planWorkerFollowUp, startWorkerFollowUp } from './worker-follow-up'
import { finishedWorkerSend, refuseFinishedRemoteWorker } from './finished-worker-mail'
import {
followThroughWorkerReport,
passHeldWorkerMail,
@@ -103,6 +104,10 @@ export const ORCHESTRATION_SEND_METHODS = [
evidence: orchestrationCompatibilityEvidence
})
}
const processIncarnation = () =>
attestedCaller?.processIncarnation ??
runtime.getTerminalProcessIncarnation(from) ??
undefined
const remoteAttachment = senderPaneKey
? db.findActiveRemoteAttachmentForPane(senderPaneKey)
: undefined
@@ -114,24 +119,26 @@ export const ORCHESTRATION_SEND_METHODS = [
from,
senderPaneKey,
remoteAttachment,
processIncarnation:
attestedCaller?.processIncarnation ??
runtime.getTerminalProcessIncarnation(from) ??
undefined,
processIncarnation: processIncarnation(),
signal
})
}
refuseFinishedRemoteWorker(db, senderPaneKey, params.to)
const runGroup =
params.to && isGroupAddress(params.to) && !params.to.toLowerCase().startsWith('@worktree:')
// Run groups validate their own audience; message scope cannot select a parent Dispatch.
const routing = resolveMessageRun(runtime, {
let routing = resolveMessageRun(runtime, {
sender,
to: params.to,
runId: runGroup ? undefined : params.run,
payload: runGroup ? undefined : params.payload,
type: params.type
})
const finished = runGroup
? undefined
: finishedWorkerSend(db, { ...sender, processIncarnation }, routing, params)
;({ params, routing } = finished ?? { params, routing })
if (
params.type === 'worker_done' &&
!isWorkerReportOutcome(parseRemoteWorkerPayload(params.payload).outcome)
@@ -271,14 +278,12 @@ export const ORCHESTRATION_SEND_METHODS = [
messageRunId,
senderPaneKey,
legacyCoordinatorRunId,
resolveProcessIncarnation: () =>
attestedCaller?.processIncarnation ??
runtime.getTerminalProcessIncarnation(from) ??
undefined,
resolveProcessIncarnation: processIncarnation,
revalidateLegacyCoordinator,
recordMutationReceipt,
markWorkerDoneMutationEffectFree,
withSendWarnings
withSendWarnings: (sent) =>
finished?.receipt(withSendWarnings(sent)) ?? withSendWarnings(sent)
})
followThroughWorkerReport(
runtime,
@@ -1,5 +1,5 @@
import type { MessageType } from '../../../orchestration/db'
import type { RunRow } from '../../../orchestration/types'
import type { DispatchContextRow, RunRow } from '../../../orchestration/types'
import type { OrcaRuntimeService } from '../../../orca-runtime'
import type { OrchestrationCallerIdentity } from '../../../orchestration/orchestration-caller-identity'
import { MESSAGE_TYPES } from '../../../orchestration/types'
@@ -20,6 +20,12 @@ export function parseMessageTypes(rawTypes: string | undefined): MessageType[] |
return types && types.length > 0 ? types : undefined
}
export type MessageRouting = {
run: RunRow | undefined
dispatchId: string | undefined
dispatch: DispatchContextRow | undefined
}
export function resolveMessageRun(
runtime: OrcaRuntimeService,
params: {
@@ -30,7 +36,7 @@ export function resolveMessageRun(
payload?: string
type?: string
}
): { run: RunRow | undefined; dispatchId: string | undefined } {
): MessageRouting {
const db = runtime.getOrchestrationDb()
let dispatchId: string | undefined
if (params.payload) {
@@ -111,7 +117,7 @@ export function resolveMessageRun(
)
}
}
return { run, dispatchId: dispatch?.id ?? dispatchId }
return { run, dispatchId: dispatch?.id ?? dispatchId, dispatch }
}
export function legacyWorkerDeliveryContract(