mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 11:24:37 +00:00
Let a finished worker keep messaging its coordinator (#38357)
* Let a finished worker keep messaging its coordinator * Tell a finished remote worker why its coordinator is out of reach * Let a finished worker's ask wait for its answer instead of starting a follow-up * Share the assignee check, keep the finished lookup in dispatch-lookup, and fix review findings * Match a finished terminal worker by its process too; cover resume after a follow-up
This commit is contained in:
@@ -79,7 +79,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
'On Windows PowerShell, quote group addresses such as --to "@all" or --to "@worktree:<id>".',
|
||||
"worker_done and heartbeat are exact-Dispatch signals and cannot target groups; omit --to to use the Dispatch's Run mailbox.",
|
||||
'worker_done requires --outcome succeeded or --outcome failed.',
|
||||
'From an active Dispatch, an omitted recipient defaults to its owning Run mailbox.',
|
||||
'From a worker, an omitted recipient defaults to its coordinator, also after it finished; mail sent after finishing, worker_done included, leaves the finished task unchanged.',
|
||||
'Use --to dispatch:<id> for attempt-specific coordinator guidance; Orca durably relays it to a connected worker server.',
|
||||
'A worker_done with the active task/dispatch IDs completes that task only from the dispatched pane. When stable pane identity is unavailable, the sender handle must exactly match the dispatch assignee; injected preambles include the correct --from value.',
|
||||
'Prefer --task-id/--dispatch-id/etc. over raw --payload JSON in worker commands; PowerShell strips JSON quotes easily.'
|
||||
@@ -239,7 +239,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
],
|
||||
identityFlagRoles: { from: 'caller' },
|
||||
notes: [
|
||||
'From an active Dispatch, a new question goes to its owning Run mailbox and ask returns at once: end your turn, and Orca wakes you with the answer as mail. --wait and --resume block until the answer instead; --timeout-ms accepts 1 to 1800000 ms per wait (default 600000 = 10 minutes, max 30 minutes), and a timeout leaves the question pending.'
|
||||
'From a worker, a new question goes to its coordinator and ask returns at once: end your turn, and Orca wakes you with the answer as mail. A worker that already finished waits for the answer instead. --wait and --resume block until the answer instead; --timeout-ms accepts 1 to 1800000 ms per wait (default 600000 = 10 minutes, max 30 minutes), and a timeout leaves the question pending.'
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -13,21 +13,22 @@ import {
|
||||
equivalentDispatchMailboxOwners
|
||||
} from '../dispatch-mailbox-selection'
|
||||
|
||||
// Hoist explicit projections so graph-publish queries hit the statement cache.
|
||||
const ACTIVE_DISPATCH_BY_HANDLE_SQL =
|
||||
// Newest-first avoids selecting a stale row when one handle has multiple active Dispatches.
|
||||
// Hoist explicit projections so graph-publish queries hit the statement cache; newest first, since
|
||||
// one handle can hold several active Dispatches.
|
||||
const assigneeSql = (where: string, status: string) =>
|
||||
`SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts
|
||||
WHERE assignee_handle = ? AND status IN ('pending', 'dispatched')
|
||||
ORDER BY rowid DESC LIMIT 1`
|
||||
const ACTIVE_DISPATCH_BY_PANE_KEY_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts
|
||||
WHERE assignee_pane_key = ? AND status IN ('pending', 'dispatched')
|
||||
ORDER BY rowid DESC LIMIT 1`
|
||||
const ACTIVE_DISPATCH_BY_PANE_SUFFIX_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts
|
||||
WHERE assignee_pane_key IS NOT NULL
|
||||
AND status IN ('pending', 'dispatched') AND instr(assignee_pane_key, ':') > 1
|
||||
AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?
|
||||
ORDER BY rowid DESC LIMIT 1`
|
||||
const LATEST_DISPATCH_BY_HANDLE_SQL = `SELECT ${DISPATCH_CONTEXT_COLUMN_LIST} FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1`
|
||||
WHERE ${where} ${status} ORDER BY rowid DESC LIMIT 1`
|
||||
const BY_PANE_SUFFIX = `assignee_pane_key IS NOT NULL AND instr(assignee_pane_key, ':') > 1
|
||||
AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?`
|
||||
const assigneeLookups = (status: string) => ({
|
||||
byHandle: assigneeSql('assignee_handle = ?', status),
|
||||
byPaneKey: assigneeSql('assignee_pane_key = ?', status),
|
||||
byPaneSuffix: assigneeSql(BY_PANE_SUFFIX, status)
|
||||
})
|
||||
const ASSIGNEE_SQL = {
|
||||
active: assigneeLookups("AND status IN ('pending', 'dispatched')"),
|
||||
latest: assigneeLookups('')
|
||||
}
|
||||
|
||||
function presentationAddressKey(column: string, address: string | null): string {
|
||||
return `${column}:${address}`
|
||||
@@ -232,16 +233,23 @@ export function isDispatchMessageSender(
|
||||
if (params.allowCanonicalDispatchHandle && params.handle === `dispatch:${dispatch.id}`) {
|
||||
return true
|
||||
}
|
||||
return isDispatchAssignee(dispatch, params.handle, params.paneKey)
|
||||
}
|
||||
|
||||
/** Whether this handle or pane was given the Dispatch, whatever its status. */
|
||||
export function isDispatchAssignee(
|
||||
dispatch: Pick<DispatchContextRow, 'assignee_handle' | 'assignee_pane_key'>,
|
||||
handle: string,
|
||||
paneKey: string | null | undefined
|
||||
): boolean {
|
||||
if (
|
||||
params.paneKey &&
|
||||
paneKey &&
|
||||
dispatch.assignee_pane_key &&
|
||||
isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey)
|
||||
isEquivalentPaneKey(dispatch.assignee_pane_key, paneKey)
|
||||
) {
|
||||
return true
|
||||
}
|
||||
return (
|
||||
params.handle === dispatch.assignee_handle && (!params.paneKey || !dispatch.assignee_pane_key)
|
||||
)
|
||||
return handle === dispatch.assignee_handle && (!paneKey || !dispatch.assignee_pane_key)
|
||||
}
|
||||
|
||||
export function findActiveDispatchForAssignee(
|
||||
@@ -249,38 +257,37 @@ export function findActiveDispatchForAssignee(
|
||||
assigneeHandle: string,
|
||||
assigneePaneKey?: string
|
||||
): DispatchContextRow | undefined {
|
||||
const byHandle = this.db.prepare(ACTIVE_DISPATCH_BY_HANDLE_SQL).get(assigneeHandle) as
|
||||
| DispatchContextRow
|
||||
| undefined
|
||||
if (byHandle) {
|
||||
return byHandle
|
||||
}
|
||||
|
||||
if (!assigneePaneKey) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const exactPane = this.db.prepare(ACTIVE_DISPATCH_BY_PANE_KEY_SQL).get(assigneePaneKey) as
|
||||
| DispatchContextRow
|
||||
| undefined
|
||||
if (exactPane) {
|
||||
return exactPane
|
||||
}
|
||||
if (!parsePaneKey(assigneePaneKey)) {
|
||||
return undefined
|
||||
}
|
||||
return this.db
|
||||
.prepare(ACTIVE_DISPATCH_BY_PANE_SUFFIX_SQL)
|
||||
.get(paneKeyMatchSuffix(assigneePaneKey)) as DispatchContextRow | undefined
|
||||
return findDispatchForAssignee(this, 'active', assigneeHandle, assigneePaneKey)
|
||||
}
|
||||
|
||||
/** The newest Dispatch, settled or not; with a pane, also after the handle was reminted. */
|
||||
export function getLatestDispatchForTerminal(
|
||||
this: OrchestrationDb,
|
||||
handle: string
|
||||
handle: string,
|
||||
paneKey?: string
|
||||
): DispatchContextRow | undefined {
|
||||
return this.db.prepare(LATEST_DISPATCH_BY_HANDLE_SQL).get(handle) as
|
||||
| DispatchContextRow
|
||||
| undefined
|
||||
return findDispatchForAssignee(this, 'latest', handle, paneKey)
|
||||
}
|
||||
|
||||
function findDispatchForAssignee(
|
||||
db: OrchestrationDb,
|
||||
kind: keyof typeof ASSIGNEE_SQL,
|
||||
handle: string,
|
||||
paneKey: string | undefined
|
||||
): DispatchContextRow | undefined {
|
||||
const sql = ASSIGNEE_SQL[kind]
|
||||
const read = (statement: string, value: string) =>
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: each statement selects the schema-pinned Dispatch projection.
|
||||
db.db.prepare(statement).get(value) as DispatchContextRow | undefined
|
||||
const byHandle = read(sql.byHandle, handle)
|
||||
if (byHandle || !paneKey) {
|
||||
return byHandle
|
||||
}
|
||||
const exactPane = read(sql.byPaneKey, paneKey)
|
||||
if (exactPane || !parsePaneKey(paneKey)) {
|
||||
return exactPane
|
||||
}
|
||||
return read(sql.byPaneSuffix, paneKeyMatchSuffix(paneKey))
|
||||
}
|
||||
|
||||
/** Helpers this Dispatch started that have not settled; their reports wake their creator. */
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import { markUnreadFollowUpMail } from './worker-follow-up-lineage'
|
||||
import type { WorkerReportOutcome, WorkerReportSettlement } from '../../types'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { AGENT_PROMPT_STALLED_ERROR } from '../../../agent-prompt-submission-verification'
|
||||
import { settleActiveDispatchesForTask } from './dispatch-completion'
|
||||
import { getActiveDispatchForTask } from './task-dispatch-reconciliation'
|
||||
import { transitionLifecycleWithDb } from '../lifecycle-transition'
|
||||
import { runLifecycleWriteTransaction } from '../lifecycle-write-transaction-runner'
|
||||
import {
|
||||
isWorkerStateIn,
|
||||
reportCorrectsUnobservedPrompt,
|
||||
SETTLEABLE_WORKER_STATES,
|
||||
UNPROVEN_WORKER_STATES
|
||||
} from '../../worker-report-admission'
|
||||
@@ -93,11 +93,8 @@ export function settleWorkerReportInTransaction(
|
||||
|
||||
const expectedDispatchStatus = params.outcome === 'succeeded' ? 'completed' : 'failed'
|
||||
const expectedTaskStatus = params.outcome === 'succeeded' ? 'completed' : 'failed'
|
||||
// A first-hand worker report can correct an unobserved prompt failure, even after circuit break; check before duplicate detection (#16095).
|
||||
const settledByUnobservedPrompt =
|
||||
(dispatch.status === 'failed' || dispatch.status === 'circuit_broken') &&
|
||||
dispatch.last_failure === AGENT_PROMPT_STALLED_ERROR &&
|
||||
task.status === 'failed'
|
||||
// Check before duplicate detection (#16095).
|
||||
const settledByUnobservedPrompt = reportCorrectsUnobservedPrompt(dispatch, task)
|
||||
const reportingWorker = this.getWorkerDispatch(params.dispatchId)
|
||||
const settledDispatchStatus = dispatch.status === 'circuit_broken' ? 'failed' : dispatch.status
|
||||
if (
|
||||
|
||||
@@ -73,22 +73,38 @@ export function markRemoteAttachmentStopUnknown(
|
||||
export function findActiveRemoteAttachmentForPane(
|
||||
this: OrchestrationDb,
|
||||
paneKey: string
|
||||
): RemoteDispatchAttachmentRow | undefined {
|
||||
return findRemoteAttachmentForPane(this, paneKey, `${potentiallyLiveRemoteAttachmentSql()} AND`)
|
||||
}
|
||||
|
||||
/** The newest remote Dispatch this pane ran, finished or not. */
|
||||
export function findLatestRemoteAttachmentForPane(
|
||||
this: OrchestrationDb,
|
||||
paneKey: string
|
||||
): RemoteDispatchAttachmentRow | undefined {
|
||||
return findRemoteAttachmentForPane(this, paneKey, '')
|
||||
}
|
||||
|
||||
function findRemoteAttachmentForPane(
|
||||
db: OrchestrationDb,
|
||||
paneKey: string,
|
||||
stateFilter: string
|
||||
): RemoteDispatchAttachmentRow | undefined {
|
||||
if (!parsePaneKey(paneKey)) {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The schema-pinned attachment projection returns RemoteDispatchAttachmentRow rows; the adapter exposes unknown.
|
||||
return this.db
|
||||
return db.db
|
||||
.prepare(
|
||||
`SELECT ${REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST} FROM remote_dispatch_attachments
|
||||
WHERE ${potentiallyLiveRemoteAttachmentSql()} AND pane_key = ?
|
||||
WHERE ${stateFilter} pane_key = ?
|
||||
ORDER BY rowid DESC LIMIT 1`
|
||||
)
|
||||
.get(paneKey) as RemoteDispatchAttachmentRow | undefined
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: The schema-pinned attachment projection returns RemoteDispatchAttachmentRow rows; the adapter exposes unknown.
|
||||
return this.db
|
||||
return db.db
|
||||
.prepare(
|
||||
`SELECT ${REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST} FROM remote_dispatch_attachments
|
||||
WHERE ${potentiallyLiveRemoteAttachmentSql()} AND pane_key IS NOT NULL
|
||||
WHERE ${stateFilter} pane_key IS NOT NULL
|
||||
AND instr(pane_key, ':') > 1
|
||||
AND ${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL} = ?
|
||||
ORDER BY rowid DESC LIMIT 1`
|
||||
@@ -101,6 +117,7 @@ export type RemoteDispatchAttachmentStopMethods = {
|
||||
settleRemoteAttachmentStop: typeof settleRemoteAttachmentStop
|
||||
markRemoteAttachmentStopUnknown: typeof markRemoteAttachmentStopUnknown
|
||||
findActiveRemoteAttachmentForPane: typeof findActiveRemoteAttachmentForPane
|
||||
findLatestRemoteAttachmentForPane: typeof findLatestRemoteAttachmentForPane
|
||||
}
|
||||
|
||||
export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }): void {
|
||||
@@ -108,6 +125,7 @@ export function attachRemoteDispatchAttachmentStop(ctor: { prototype: object }):
|
||||
beginRemoteAttachmentStop,
|
||||
settleRemoteAttachmentStop,
|
||||
markRemoteAttachmentStopUnknown,
|
||||
findActiveRemoteAttachmentForPane
|
||||
findActiveRemoteAttachmentForPane,
|
||||
findLatestRemoteAttachmentForPane
|
||||
})
|
||||
}
|
||||
|
||||
@@ -19,15 +19,12 @@ export function createQuestion(
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
this.requireRun(params.runId)
|
||||
// A finished Dispatch still asks: its worker keeps talking to the coordinator after reporting.
|
||||
const dispatch = this.getDispatchContextById(params.dispatchId)
|
||||
if (
|
||||
!dispatch ||
|
||||
dispatch.run_id !== params.runId ||
|
||||
(dispatch.status !== 'pending' && dispatch.status !== 'dispatched')
|
||||
) {
|
||||
if (!dispatch || dispatch.run_id !== params.runId) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
`Dispatch ${params.dispatchId} is not active in Run ${params.runId}.`
|
||||
`Dispatch ${params.dispatchId} is not in Run ${params.runId}.`
|
||||
)
|
||||
}
|
||||
const message = this.insertMessage({
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { OrchestrationError } from './orchestration-error'
|
||||
import type { WorkerDispatchState } from './types'
|
||||
import { AGENT_PROMPT_STALLED_ERROR } from '../agent-prompt-submission-verification'
|
||||
import type { DispatchContextRow, TaskRow, WorkerDispatchState } from './types'
|
||||
|
||||
// Unknown start/stop cannot prove exit; the worker's report outranks uncertainty (docs/reference/ssh-execution-boundary.md).
|
||||
export const UNPROVEN_WORKER_STATES = [
|
||||
@@ -52,3 +53,15 @@ export function assertWorkerCanReport(args: Parameters<typeof workerReportRefusa
|
||||
throw new OrchestrationError(refusal.code, refusal.reason)
|
||||
}
|
||||
}
|
||||
|
||||
/** A first-hand worker report can correct an unobserved prompt failure, even after circuit break. */
|
||||
export function reportCorrectsUnobservedPrompt(
|
||||
dispatch: Pick<DispatchContextRow, 'status' | 'last_failure'>,
|
||||
task: Pick<TaskRow, 'status'>
|
||||
): boolean {
|
||||
return (
|
||||
(dispatch.status === 'failed' || dispatch.status === 'circuit_broken') &&
|
||||
dispatch.last_failure === AGENT_PROMPT_STALLED_ERROR &&
|
||||
task.status === 'failed'
|
||||
)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
import { resolveOrchestrationParty } from '../../../../orchestration/orchestration-party'
|
||||
import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence'
|
||||
import { assertWorkerCanReport } from '../../../../orchestration/worker-report-admission'
|
||||
import { finishedWorkerDispatch, refuseFinishedRemoteWorker } from './finished-worker-mail'
|
||||
import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup'
|
||||
import {
|
||||
askedQuestionReceipt,
|
||||
waitForQuestionAnswer
|
||||
@@ -68,25 +70,42 @@ export const ORCHESTRATION_ASK_METHODS = [
|
||||
workerState: remoteAttachment.state
|
||||
})
|
||||
}
|
||||
const activeDispatch = db.getActiveDispatchForIdentity(from, paneKey)
|
||||
if (!activeDispatch) {
|
||||
const workingDispatch = db.getActiveDispatchForIdentity(from, paneKey)
|
||||
// A finished worker still asks its coordinator; the question changes nothing about its Task.
|
||||
const askingDispatch =
|
||||
workingDispatch ??
|
||||
finishedWorkerDispatch(
|
||||
db,
|
||||
{
|
||||
address: from,
|
||||
paneKey: paneKey ?? null,
|
||||
orcaSessionId: orchestrationCaller?.orcaSessionId ?? null,
|
||||
processIncarnation: () => runtime.getTerminalProcessIncarnation(from) ?? undefined
|
||||
},
|
||||
{ run: undefined, dispatchId: undefined, dispatch: undefined },
|
||||
'question'
|
||||
)
|
||||
if (!askingDispatch) {
|
||||
refuseFinishedRemoteWorker(db, paneKey, undefined)
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
'ask requires an active supervised Dispatch.'
|
||||
)
|
||||
}
|
||||
assertWorkerCanReport({
|
||||
dispatchId: activeDispatch.id,
|
||||
from,
|
||||
workerState: db.getWorkerDispatch(activeDispatch.id)?.state,
|
||||
processCurrent:
|
||||
!activeDispatch.process_incarnation ||
|
||||
db.isDispatchProcessCurrent({
|
||||
dispatchId: activeDispatch.id,
|
||||
paneKey: paneKey ?? null,
|
||||
processIncarnation: runtime.getTerminalProcessIncarnation(from)
|
||||
})
|
||||
})
|
||||
if (workingDispatch) {
|
||||
assertWorkerCanReport({
|
||||
dispatchId: workingDispatch.id,
|
||||
from,
|
||||
workerState: db.getWorkerDispatch(workingDispatch.id)?.state,
|
||||
processCurrent:
|
||||
!workingDispatch.process_incarnation ||
|
||||
db.isDispatchProcessCurrent({
|
||||
dispatchId: workingDispatch.id,
|
||||
paneKey: paneKey ?? null,
|
||||
processIncarnation: runtime.getTerminalProcessIncarnation(from)
|
||||
})
|
||||
})
|
||||
}
|
||||
const options =
|
||||
params.options
|
||||
?.split(',')
|
||||
@@ -94,27 +113,29 @@ export const ORCHESTRATION_ASK_METHODS = [
|
||||
.filter(Boolean) ?? []
|
||||
let question = params.resume ? db.getQuestion(params.resume) : undefined
|
||||
if (params.resume) {
|
||||
if (!question || question.dispatch_id !== activeDispatch.id) {
|
||||
// Its own question from a finished Dispatch stays resumable after a follow-up starts.
|
||||
const askedBy = question && db.getDispatchContextById(question.dispatch_id)
|
||||
if (!question || !askedBy || !isDispatchAssignee(askedBy, from, paneKey)) {
|
||||
throw new OrchestrationError(
|
||||
'question_not_found',
|
||||
`Question ${params.resume} does not belong to this active Dispatch.`
|
||||
)
|
||||
}
|
||||
} else {
|
||||
const run = db.getRun(activeDispatch.run_id)
|
||||
const run = db.getRun(askingDispatch.run_id)
|
||||
if (!run || run.legacy === 1) {
|
||||
throw new OrchestrationError(
|
||||
'run_not_found',
|
||||
`Run ${activeDispatch.run_id} was not found.`
|
||||
`Run ${askingDispatch.run_id} was not found.`
|
||||
)
|
||||
}
|
||||
if (params.run && params.run !== run.id) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_run_mismatch',
|
||||
`Dispatch ${activeDispatch.id} belongs to Run ${run.id}, not ${params.run}.`
|
||||
`Dispatch ${askingDispatch.id} belongs to Run ${run.id}, not ${params.run}.`
|
||||
)
|
||||
}
|
||||
const reportTo = dispatchReportMailbox(db, activeDispatch)
|
||||
const reportTo = dispatchReportMailbox(db, askingDispatch)
|
||||
if (
|
||||
params.to &&
|
||||
params.to !== `run:${run.id}` &&
|
||||
@@ -124,14 +145,14 @@ export const ORCHESTRATION_ASK_METHODS = [
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_run_mismatch',
|
||||
`ask from Dispatch ${activeDispatch.id} must target its owning Run ${run.id}.`
|
||||
`ask from Dispatch ${askingDispatch.id} must target its owning Run ${run.id}.`
|
||||
)
|
||||
}
|
||||
// An explicit Run or coordinator target still reaches the Run; otherwise the asker's creator.
|
||||
const to = params.to && params.to !== reportTo ? `run:${run.id}` : reportTo
|
||||
const created = db.createQuestion({
|
||||
runId: run.id,
|
||||
dispatchId: activeDispatch.id,
|
||||
dispatchId: askingDispatch.id,
|
||||
askerHandle: from,
|
||||
question: params.question as string,
|
||||
options,
|
||||
@@ -142,7 +163,8 @@ export const ORCHESTRATION_ASK_METHODS = [
|
||||
}
|
||||
|
||||
const questionId = question.message_id
|
||||
if (!params.resume && params.wait === false) {
|
||||
// A finished worker has no turn for an answer to wake, so it waits for it here.
|
||||
if (!params.resume && params.wait === false && workingDispatch) {
|
||||
return askedQuestionReceipt(questionId, recordMutationReceipt)
|
||||
}
|
||||
return waitForQuestionAnswer({
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createOrchestrationRpcHarness } from '../rpc-test-harness'
|
||||
import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture'
|
||||
|
||||
describe('a worker after its worker_done', () => {
|
||||
const harness = createOrchestrationRpcHarness()
|
||||
afterEach(() => harness.cleanup())
|
||||
|
||||
it('still reaches its coordinator with every message, and none changes the finished Task', async () => {
|
||||
const { db, runtime, ctx, activeRunId } = harness.setup()
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = createRootDispatch(
|
||||
db,
|
||||
task.id,
|
||||
'term_worker',
|
||||
'tab_worker:leaf_worker',
|
||||
undefined,
|
||||
'runtime_test:term_worker:1'
|
||||
)
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === 'term_worker'
|
||||
? 'tab_worker:leaf_worker'
|
||||
: handle === 'term_coord'
|
||||
? harness.coordinatorPaneKey
|
||||
: null
|
||||
)
|
||||
const send = (params: Record<string, unknown>) =>
|
||||
harness.call('orchestration.send', { from: 'term_worker', ...params }, ctx)
|
||||
const report = (outcome: string) =>
|
||||
JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome })
|
||||
|
||||
await send({ type: 'worker_done', subject: 'Done', payload: report('succeeded') })
|
||||
const settled = db.getTask(task.id)
|
||||
|
||||
await send({ subject: 'No recipient' })
|
||||
await send({ to: `dispatch:${dispatch.id}`, subject: 'To my own Dispatch' })
|
||||
const again = await send({
|
||||
type: 'worker_done',
|
||||
subject: 'Second report',
|
||||
payload: report('failed')
|
||||
})
|
||||
// It has no turn for a later answer to wake, so even a returning ask waits for the answer.
|
||||
const asking = harness.call(
|
||||
'orchestration.ask',
|
||||
{ from: 'term_worker', question: 'Anything else?', wait: false, timeoutMs: 5_000 },
|
||||
ctx
|
||||
)
|
||||
await vi.waitFor(() => expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(5))
|
||||
const question = db.getUnreadMessages(`run:${activeRunId}`)[4]
|
||||
|
||||
expect(again).toMatchObject({ lifecycle: { action: 'settled', outcome: 'succeeded' } })
|
||||
expect(db.getTask(task.id)).toEqual(settled)
|
||||
expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed')
|
||||
expect(
|
||||
db.getUnreadMessages(`run:${activeRunId}`).map((message) => [message.type, message.subject])
|
||||
).toEqual([
|
||||
['worker_done', 'Done'],
|
||||
['status', 'No recipient'],
|
||||
['status', 'To my own Dispatch'],
|
||||
['status', 'Second report'],
|
||||
['question', 'Question']
|
||||
])
|
||||
await harness.call(
|
||||
'orchestration.reply',
|
||||
{ from: 'term_coord', id: question.id, body: 'Nothing else.' },
|
||||
ctx
|
||||
)
|
||||
expect(await asking).toMatchObject({ answer: 'Nothing else.', timedOut: false })
|
||||
})
|
||||
|
||||
it("does not take a new process in the finished worker's pane for that worker", async () => {
|
||||
const { db, runtime, ctx } = harness.setup()
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = createRootDispatch(
|
||||
db,
|
||||
task.id,
|
||||
'term_worker',
|
||||
'tab_worker:leaf_worker',
|
||||
undefined,
|
||||
'runtime_test:term_worker:1'
|
||||
)
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === 'term_worker' ? 'tab_worker:leaf_worker' : null
|
||||
)
|
||||
await harness.call(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: 'term_worker',
|
||||
type: 'worker_done',
|
||||
subject: 'Done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' })
|
||||
},
|
||||
ctx
|
||||
)
|
||||
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime_test:term_worker:2')
|
||||
|
||||
await expect(
|
||||
harness.call('orchestration.send', { from: 'term_worker', subject: 'Not mine' }, ctx)
|
||||
).rejects.toMatchObject({ code: 'run_required' })
|
||||
})
|
||||
|
||||
it('resumes its own question after a follow-up starts on the same worker', async () => {
|
||||
const { db, runtime, ctx } = harness.setup()
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = createRootDispatch(db, task.id, 'term_worker', 'tab_worker:leaf_worker')
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === 'term_worker'
|
||||
? 'tab_worker:leaf_worker'
|
||||
: handle === 'term_coord'
|
||||
? harness.coordinatorPaneKey
|
||||
: null
|
||||
)
|
||||
await harness.call(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: 'term_worker',
|
||||
type: 'worker_done',
|
||||
subject: 'Done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' })
|
||||
},
|
||||
ctx
|
||||
)
|
||||
const asked = await harness.call(
|
||||
'orchestration.ask',
|
||||
{ from: 'term_worker', question: 'Anything else?', timeoutMs: 1 },
|
||||
ctx
|
||||
)
|
||||
expect(asked).toMatchObject({ timedOut: true })
|
||||
const questionId =
|
||||
asked && typeof asked === 'object' && 'messageId' in asked ? asked.messageId : ''
|
||||
createRootDispatch(
|
||||
db,
|
||||
db.createTask({ spec: 'more' }).id,
|
||||
'term_worker',
|
||||
'tab_worker:leaf_worker'
|
||||
)
|
||||
await harness.call(
|
||||
'orchestration.reply',
|
||||
{ from: 'term_coord', id: questionId, body: 'Nothing else.' },
|
||||
ctx
|
||||
)
|
||||
|
||||
await expect(
|
||||
harness.call('orchestration.ask', { from: 'term_worker', resume: questionId }, ctx)
|
||||
).resolves.toMatchObject({ answer: 'Nothing else.' })
|
||||
})
|
||||
|
||||
it('tells a finished worker on a paired host that its coordinator is out of reach', async () => {
|
||||
const { db, runtime, ctx } = harness.setup(false)
|
||||
db.createRemoteDispatchAttachment({
|
||||
dispatchId: 'ctx_remote',
|
||||
runId: 'run_home',
|
||||
taskId: 'task_remote',
|
||||
homePeerFingerprint: 'peer_fp',
|
||||
protocolVersion: 1,
|
||||
runtimeEpoch: 'epoch_1',
|
||||
mutationReceipt: {
|
||||
callerFingerprint: 'peer_fp',
|
||||
requestId: 'attach_1',
|
||||
method: 'orchestration.federationAttachStart',
|
||||
payloadHash: 'attach_payload'
|
||||
}
|
||||
})
|
||||
db.prepareRemoteAttachmentAuthority({
|
||||
dispatchId: 'ctx_remote',
|
||||
paneKey: 'tab_worker:leaf_worker',
|
||||
processIncarnation: 'runtime_test:term_worker:1',
|
||||
worktreeId: 'folder_workspace',
|
||||
terminalHandle: 'term_worker',
|
||||
setupState: 'not_applicable',
|
||||
effects: []
|
||||
})
|
||||
db.markRemoteAttachmentReady('ctx_remote')
|
||||
db.settleRemoteAttachmentInRelayTransaction('ctx_remote', 'succeeded')
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === 'term_worker' ? 'tab_worker:leaf_worker' : null
|
||||
)
|
||||
|
||||
for (const [method, params] of [
|
||||
['orchestration.send', { subject: 'Later' }],
|
||||
['orchestration.send', { to: 'dispatch:ctx_remote', subject: 'To my own Dispatch' }],
|
||||
['orchestration.ask', { question: 'Anything else?', wait: false }]
|
||||
] as const) {
|
||||
await expect(
|
||||
harness.call(method, { from: 'term_worker', ...params }, ctx)
|
||||
).rejects.toMatchObject({ code: 'run_home_unreachable' })
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,101 @@
|
||||
import type { OrchestrationDb } from '../../../../orchestration/db'
|
||||
import type { OrchestrationCallerIdentity } from '../../../../orchestration/orchestration-caller-identity'
|
||||
import type { DispatchContextRow } from '../../../../orchestration/types'
|
||||
import { dispatchReportMailbox } from '../../../../orchestration/dispatch-report-mailbox'
|
||||
import { reportCorrectsUnobservedPrompt } from '../../../../orchestration/worker-report-admission'
|
||||
import { isDispatchAssignee } from '../../../../orchestration/db/dispatch-context/dispatch-lookup'
|
||||
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
|
||||
import { isDispatchMutationMessageType } from '../schemas'
|
||||
import type { MessageRouting } from '../routing'
|
||||
|
||||
/** A terminal also proves its process, so a new process in the same pane is not taken for the old worker;
|
||||
* a session caller is already bound to its own identity. */
|
||||
type FinishedSender = Pick<OrchestrationCallerIdentity, 'address' | 'paneKey' | 'orcaSessionId'> & {
|
||||
processIncarnation: () => string | undefined
|
||||
}
|
||||
|
||||
/** The Dispatch a worker finished and is still talking about: the one its message names, or, naming none
|
||||
* and coordinating nothing, the last one it was given. What it sends there reaches that Dispatch's coordinator. */
|
||||
export function finishedWorkerDispatch(
|
||||
db: OrchestrationDb,
|
||||
sender: FinishedSender,
|
||||
routing: Pick<MessageRouting, 'run' | 'dispatchId' | 'dispatch'>,
|
||||
type: string | undefined
|
||||
): DispatchContextRow | undefined {
|
||||
// A late heartbeat is already dropped quietly; it says nothing the coordinator needs.
|
||||
if (type === 'heartbeat' || (!routing.dispatchId && routing.run)) {
|
||||
return undefined
|
||||
}
|
||||
const dispatch = routing.dispatchId
|
||||
? routing.dispatch
|
||||
: db.getLatestDispatchForTerminal(sender.address, sender.paneKey ?? undefined)
|
||||
if (
|
||||
!dispatch ||
|
||||
dispatch.status === 'pending' ||
|
||||
dispatch.status === 'dispatched' ||
|
||||
!isDispatchAssignee(dispatch, sender.address, sender.paneKey) ||
|
||||
(!sender.orcaSessionId &&
|
||||
dispatch.process_incarnation &&
|
||||
!db.isDispatchProcessCurrent({
|
||||
dispatchId: dispatch.id,
|
||||
paneKey: sender.paneKey,
|
||||
processIncarnation: sender.processIncarnation() ?? null
|
||||
})) ||
|
||||
db.getFederatedDispatch(dispatch.id)
|
||||
) {
|
||||
return undefined
|
||||
}
|
||||
const task = type === 'worker_done' ? db.getTask(dispatch.task_id) : undefined
|
||||
return task && reportCorrectsUnobservedPrompt(dispatch, task) ? undefined : dispatch
|
||||
}
|
||||
|
||||
/** Mail from a finished worker is ordinary mail to its coordinator, so no report or escalation reopens the Task. */
|
||||
export function finishedWorkerSend<P extends { to?: string; type?: string }>(
|
||||
db: OrchestrationDb,
|
||||
sender: FinishedSender,
|
||||
routing: MessageRouting,
|
||||
params: P
|
||||
):
|
||||
| { params: P; routing: MessageRouting; receipt: <T extends object>(receipt: T) => T }
|
||||
| undefined {
|
||||
const finished = finishedWorkerDispatch(db, sender, routing, params.type)
|
||||
if (!finished) {
|
||||
return undefined
|
||||
}
|
||||
const ownDispatch = !params.to || params.to === `dispatch:${finished.id}`
|
||||
return {
|
||||
params: {
|
||||
...params,
|
||||
type: isDispatchMutationMessageType(params.type) ? 'status' : params.type,
|
||||
to: ownDispatch ? dispatchReportMailbox(db, finished) : params.to
|
||||
},
|
||||
routing: { run: db.getRun(finished.run_id), dispatchId: undefined, dispatch: undefined },
|
||||
// Older CLIs confirm a worker_done only by a verdict; the Task's recorded one stands.
|
||||
receipt: (receipt) =>
|
||||
params.type === 'worker_done'
|
||||
? {
|
||||
...receipt,
|
||||
lifecycle: {
|
||||
action: 'settled',
|
||||
outcome: finished.status === 'completed' ? 'succeeded' : 'failed'
|
||||
},
|
||||
next: 'Delivered to your coordinator as a message; your finished task is unchanged. End your turn.'
|
||||
}
|
||||
: receipt
|
||||
}
|
||||
}
|
||||
|
||||
/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. */
|
||||
export function refuseFinishedRemoteWorker(
|
||||
db: OrchestrationDb,
|
||||
paneKey: string | undefined,
|
||||
to: string | undefined
|
||||
): void {
|
||||
const attachment = paneKey ? db.findLatestRemoteAttachmentForPane(paneKey) : undefined
|
||||
if (attachment && (!to || to === `dispatch:${attachment.dispatch_id}`)) {
|
||||
throw new OrchestrationError(
|
||||
'run_home_unreachable',
|
||||
`Dispatch ${attachment.dispatch_id} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import { sendFederatedControlMail } from './send-control-mail'
|
||||
import { orchestrationCallerIdentity } from '../runs/run-scope'
|
||||
import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence'
|
||||
import { planWorkerFollowUp, startWorkerFollowUp } from './worker-follow-up'
|
||||
import { finishedWorkerSend, refuseFinishedRemoteWorker } from './finished-worker-mail'
|
||||
import {
|
||||
followThroughWorkerReport,
|
||||
passHeldWorkerMail,
|
||||
@@ -103,6 +104,10 @@ export const ORCHESTRATION_SEND_METHODS = [
|
||||
evidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
}
|
||||
const processIncarnation = () =>
|
||||
attestedCaller?.processIncarnation ??
|
||||
runtime.getTerminalProcessIncarnation(from) ??
|
||||
undefined
|
||||
const remoteAttachment = senderPaneKey
|
||||
? db.findActiveRemoteAttachmentForPane(senderPaneKey)
|
||||
: undefined
|
||||
@@ -114,24 +119,26 @@ export const ORCHESTRATION_SEND_METHODS = [
|
||||
from,
|
||||
senderPaneKey,
|
||||
remoteAttachment,
|
||||
processIncarnation:
|
||||
attestedCaller?.processIncarnation ??
|
||||
runtime.getTerminalProcessIncarnation(from) ??
|
||||
undefined,
|
||||
processIncarnation: processIncarnation(),
|
||||
signal
|
||||
})
|
||||
}
|
||||
|
||||
refuseFinishedRemoteWorker(db, senderPaneKey, params.to)
|
||||
const runGroup =
|
||||
params.to && isGroupAddress(params.to) && !params.to.toLowerCase().startsWith('@worktree:')
|
||||
// Run groups validate their own audience; message scope cannot select a parent Dispatch.
|
||||
const routing = resolveMessageRun(runtime, {
|
||||
let routing = resolveMessageRun(runtime, {
|
||||
sender,
|
||||
to: params.to,
|
||||
runId: runGroup ? undefined : params.run,
|
||||
payload: runGroup ? undefined : params.payload,
|
||||
type: params.type
|
||||
})
|
||||
const finished = runGroup
|
||||
? undefined
|
||||
: finishedWorkerSend(db, { ...sender, processIncarnation }, routing, params)
|
||||
;({ params, routing } = finished ?? { params, routing })
|
||||
if (
|
||||
params.type === 'worker_done' &&
|
||||
!isWorkerReportOutcome(parseRemoteWorkerPayload(params.payload).outcome)
|
||||
@@ -271,14 +278,12 @@ export const ORCHESTRATION_SEND_METHODS = [
|
||||
messageRunId,
|
||||
senderPaneKey,
|
||||
legacyCoordinatorRunId,
|
||||
resolveProcessIncarnation: () =>
|
||||
attestedCaller?.processIncarnation ??
|
||||
runtime.getTerminalProcessIncarnation(from) ??
|
||||
undefined,
|
||||
resolveProcessIncarnation: processIncarnation,
|
||||
revalidateLegacyCoordinator,
|
||||
recordMutationReceipt,
|
||||
markWorkerDoneMutationEffectFree,
|
||||
withSendWarnings
|
||||
withSendWarnings: (sent) =>
|
||||
finished?.receipt(withSendWarnings(sent)) ?? withSendWarnings(sent)
|
||||
})
|
||||
followThroughWorkerReport(
|
||||
runtime,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { MessageType } from '../../../orchestration/db'
|
||||
import type { RunRow } from '../../../orchestration/types'
|
||||
import type { DispatchContextRow, RunRow } from '../../../orchestration/types'
|
||||
import type { OrcaRuntimeService } from '../../../orca-runtime'
|
||||
import type { OrchestrationCallerIdentity } from '../../../orchestration/orchestration-caller-identity'
|
||||
import { MESSAGE_TYPES } from '../../../orchestration/types'
|
||||
@@ -20,6 +20,12 @@ export function parseMessageTypes(rawTypes: string | undefined): MessageType[] |
|
||||
return types && types.length > 0 ? types : undefined
|
||||
}
|
||||
|
||||
export type MessageRouting = {
|
||||
run: RunRow | undefined
|
||||
dispatchId: string | undefined
|
||||
dispatch: DispatchContextRow | undefined
|
||||
}
|
||||
|
||||
export function resolveMessageRun(
|
||||
runtime: OrcaRuntimeService,
|
||||
params: {
|
||||
@@ -30,7 +36,7 @@ export function resolveMessageRun(
|
||||
payload?: string
|
||||
type?: string
|
||||
}
|
||||
): { run: RunRow | undefined; dispatchId: string | undefined } {
|
||||
): MessageRouting {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
let dispatchId: string | undefined
|
||||
if (params.payload) {
|
||||
@@ -111,7 +117,7 @@ export function resolveMessageRun(
|
||||
)
|
||||
}
|
||||
}
|
||||
return { run, dispatchId: dispatch?.id ?? dispatchId }
|
||||
return { run, dispatchId: dispatch?.id ?? dispatchId, dispatch }
|
||||
}
|
||||
|
||||
export function legacyWorkerDeliveryContract(
|
||||
|
||||
Reference in New Issue
Block a user