From bfd1723ecbd4d7ae5d6100fcf99374f853bb6eb8 Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:32:27 -0800 Subject: [PATCH] fix(orchestration): let a restart-reissued worker's plain check take its answer, and a working local owner outrank finished remote history A recovered federated worker's default (nonblocking) ask delivered its answer to Dispatch mail, but the plain check its wake types with the stale env handle failed the attachment's process fence and left the answer unread. check now resolves the caller through the same exact-incarnation recovery as ask and the reports, at entry and at each revalidation, so replacement and pane turnover still refuse. A worker given local work after its remote Dispatch finished had its default worker_done refused as run_home_unreachable by the finished attachment. The refusal now yields to a working local Dispatch and still refuses mail that names the finished remote Dispatch. The recovered reporter's admission re-read also pins the entry incarnation. --- .../reports/reminted-worker-reporter.ts | 18 +- .../orchestration/messaging/ask-methods.ts | 4 +- .../orchestration/messaging/check-methods.ts | 14 +- .../orchestration/messaging/check-worker.ts | 6 +- .../messaging/finished-worker-mail.ts | 27 ++- .../reminted-worker-report-continuity.test.ts | 213 ++++++++++++++++++ .../orchestration/messaging/send-methods.ts | 9 +- 7 files changed, 267 insertions(+), 24 deletions(-) diff --git a/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts b/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts index d4239ab28d1..fc791c878d2 100644 --- a/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts +++ b/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts @@ -153,25 +153,33 @@ export function resolveWorkerReporterIdentity( db: OrchestrationDb, args: { from: string - attested: - | Pick + /** The hook-attested caller, which speaks for `from` only when it names that same handle. */ + authority: + | Pick< + OrchestrationCompatibilityCallerAuthority, + 'terminalHandle' | 'paneKey' | 'processIncarnation' + > | undefined hasSessionCaller: boolean evidence: OrchestrationCompatibilityEvidence | undefined } ): { paneKey: string | null; processIncarnation: () => string | null } { - const { from, attested } = args + const { from } = args + // Why: attested hook identity survives graph remount; caller params never supply lifecycle authority. + const attested = args.authority?.terminalHandle === from ? args.authority : undefined const declaredPaneKey = attested?.paneKey ?? runtime.getTerminalPaneKey(from) const remint = () => args.hasSessionCaller || attested || declaredPaneKey ? null : resolveRemintedWorkerReporter(runtime, db, { from, evidence: args.evidence }) const reminted = remint() - // The re-read must still speak for the same Dispatch and pane that routing already used. + // The re-read must still speak for the same Dispatch, pane and process that routing already used, so a + // record rewritten to another incarnation after entry cannot carry the request over to it. const remintedAgain = (entry: RemintedWorkerReporter): string | null => { const again = remint() return again?.dispatchId === entry.dispatchId && - isEquivalentPaneKey(again.paneKey, entry.paneKey) + isEquivalentPaneKey(again.paneKey, entry.paneKey) && + again.processIncarnation === entry.processIncarnation ? again.processIncarnation : null } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts index 972bc41953e..af07176e9fc 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts @@ -56,7 +56,7 @@ export const ORCHESTRATION_ASK_METHODS = [ // A runtime restart can reissue a surviving worker PTY's handle; recover it by exact incarnation. const reporter = resolveWorkerReporterIdentity(runtime, db, { from, - attested: undefined, + authority: undefined, hasSessionCaller: Boolean(orchestrationCaller), evidence: orchestrationCompatibilityEvidence }) @@ -101,7 +101,7 @@ export const ORCHESTRATION_ASK_METHODS = [ 'question' ) if (!askingDispatch) { - refuseFinishedRemoteWorker(db, paneKey, undefined) + refuseFinishedRemoteWorker(db, { address: from, paneKey }, undefined) throw new OrchestrationError( 'dispatch_inactive', 'ask requires an active supervised Dispatch.' diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts index 11af8dd3c25..493039f2750 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts @@ -10,6 +10,7 @@ import { withRunCheckNext } from '../receipt-next' import { orchestrationSkillRecoveryData } from '../../../../../../shared/orchestration/orchestration-rpc-contract' import { hasRunBindingKey } from '../../../../orchestration/addressing/orchestration-caller-identity' import { orchestrationCallerIdentity } from '../runs/run-scope' +import { resolveWorkerReporterIdentity } from '../../../../orchestration/workers/reports/reminted-worker-reporter' import { callerHoldsDispatchPane, dispatchFenced, @@ -44,11 +45,19 @@ export const ORCHESTRATION_CHECK_METHODS = [ const handle = params.terminal ?? 'unknown' const typeFilter = parseMessageTypes(params.types) + // A restart can reissue a surviving worker PTY's handle; the woken worker still checks with its env + // handle. Recover it by the Dispatch's exact recorded incarnation, as its ask and reports were. + const reporter = resolveWorkerReporterIdentity(runtime, db, { + from: handle, + authority: undefined, + hasSessionCaller: Boolean(orchestrationCaller), + evidence: orchestrationCompatibilityEvidence + }) const caller = orchestrationCallerIdentity(runtime, { handle, session: orchestrationCaller, // Why: a live runtime handle is authoritative; pane metadata is only the restart fallback. - paneKey: runtime.getTerminalPaneKey(handle) ?? params.terminalPaneKey + paneKey: reporter.paneKey ?? params.terminalPaneKey }) const paneKey = caller.paneKey ?? undefined const boundRun = hasRunBindingKey(caller) ? db.getCurrentRunForCoordinator(caller) : undefined @@ -94,7 +103,7 @@ export const ORCHESTRATION_CHECK_METHODS = [ !db.isRemoteAttachmentProcessCurrent({ dispatchId: remoteAttachment.dispatch_id, paneKey: paneKey ?? null, - processIncarnation: runtime.getTerminalProcessIncarnation(handle) + processIncarnation: reporter.processIncarnation() }) ) { throw new OrchestrationError( @@ -114,6 +123,7 @@ export const ORCHESTRATION_CHECK_METHODS = [ signal, activeDispatch, remoteAttachment, + processIncarnation: reporter.processIncarnation, recordMutationReceipt, acknowledgeOnly: !params.wait }) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts index 8ae22f5fa61..559f0d599d4 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts @@ -28,6 +28,8 @@ export async function checkWorkerMailbox(args: { signal: AbortSignal | undefined activeDispatch: ActiveDispatch | undefined remoteAttachment: RemoteAttachment | undefined + /** The caller's process, re-read at each fence; a restart-reissued handle supplies its recovered one. */ + processIncarnation?: () => string | null wakeTypes?: MessageType[] revalidateConsumer?: () => void deferDelivery?: () => boolean @@ -100,7 +102,9 @@ export async function checkWorkerMailbox(args: { db.isRemoteAttachmentProcessCurrent({ dispatchId: current.dispatch_id, paneKey, - processIncarnation: runtime.getTerminalProcessIncarnation(handle) + processIncarnation: args.processIncarnation + ? args.processIncarnation() + : runtime.getTerminalProcessIncarnation(handle) }) ) { return diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts index 791c8a2905d..789c20e2b16 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts @@ -85,17 +85,30 @@ export function finishedWorkerSend

( } } -/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. */ +/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. + * A worker since given local work is no longer that finished worker: its default mail goes to its + * working Dispatch, so only mail naming the finished remote Dispatch is refused. */ export function refuseFinishedRemoteWorker( db: OrchestrationDb, - paneKey: string | undefined, + sender: { address: string; paneKey: string | undefined }, to: string | undefined ): void { + const { address, paneKey } = sender const attachment = paneKey ? db.findLatestRemoteAttachmentForPane(paneKey) : undefined - if (attachment && (!to || to === `dispatch:${attachment.dispatch_id}`)) { - throw new OrchestrationError( - 'run_home_unreachable', - `Dispatch ${attachment.dispatch_id} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.` - ) + if (!attachment) { + return + } + if (to === `dispatch:${attachment.dispatch_id}`) { + throw finishedRemoteUnreachable(attachment.dispatch_id) + } + if (!to && !db.getActiveDispatchForIdentity(address, paneKey)) { + throw finishedRemoteUnreachable(attachment.dispatch_id) } } + +function finishedRemoteUnreachable(dispatchId: string): OrchestrationError { + return new OrchestrationError( + 'run_home_unreachable', + `Dispatch ${dispatchId} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.` + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts index d874561208c..395e611d265 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts @@ -6,6 +6,7 @@ import type { OrchestrationDb } from '../../../../orchestration/db/orchestration import type { OrcaRuntimeService } from '../../../../orca-runtime' import { createOrchestrationRpcHarness } from '../rpc-test-harness' import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture' +import { resolveWorkerReporterIdentity } from '../../../../orchestration/workers/reports/reminted-worker-reporter' const STALE_HANDLE = 'term_exported-before-restart' const REMINTED_HANDLE = 'term_reissued-after-restart' @@ -116,6 +117,48 @@ describe('restart-reissued worker handle across finished and federated Dispatche workerEnv() ) + /** The plain `orchestration check` an Orca wake types: the env handle and pane, nothing else. */ + const check = (from: string = STALE_HANDLE, env: RpcContext = workerEnv()) => + harness.call('orchestration.check', { terminal: from, terminalPaneKey: WORKER_PANE }, env) + + const askedId = (asked: unknown): string => + asked && typeof asked === 'object' && 'messageId' in asked ? String(asked.messageId) : '' + + const checkedBodies = (checked: unknown): string[] => + checked && + typeof checked === 'object' && + 'messages' in checked && + Array.isArray(checked.messages) + ? checked.messages.map((message: { body?: unknown }) => String(message.body)) + : [] + + /** The Run home answers the worker's question; the answer lands in its Dispatch mail. */ + const importRemoteAnswer = (questionId: string) => + harness.call( + 'orchestration.federationImport', + { + dispatchId: REMOTE_DISPATCH, + items: [ + { + dispatch_id: REMOTE_DISPATCH, + direction: 'to_worker', + sequence: 1, + message_id: 'relay_answer', + kind: 'reply', + payload: JSON.stringify({ + questionId, + answerMessageId: 'msg_remote_answer', + body: 'Approved' + }) + } + ] + }, + { ...ctx, authenticatedCallerFingerprint: 'home_peer' } + ) + + const remoteAnswerUnread = () => + db.getUnreadMessages(`dispatch:${REMOTE_DISPATCH}`).some((m) => m.body === 'Approved') + const remoteRelayKinds = () => db .listFederationRelay({ dispatchId: REMOTE_DISPATCH, direction: 'to_home', afterSequence: 0 }) @@ -158,6 +201,42 @@ describe('restart-reissued worker handle across finished and federated Dispatche }) }) + describe('working local worker', () => { + it('consumes its default question answer with the plain check its wake types', async () => { + setupLocalWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + expect(asked).toMatchObject({ pending: true }) + await harness.call( + 'orchestration.reply', + { id: askedId(asked), body: 'Approved locally', from: 'term_coord' }, + ctx + ) + + expect(checkedBodies(await check())).toContain('Approved locally') + }) + }) + + describe('admission re-read', () => { + it('refuses a Dispatch record rewritten to another incarnation after entry', () => { + const { dispatchId } = setupLocalWorker() + const reporter = resolveWorkerReporterIdentity(runtime, db, { + from: STALE_HANDLE, + authority: undefined, + hasSessionCaller: false, + evidence: workerEnv().orchestrationCompatibilityEvidence + }) + expect(reporter.processIncarnation()).toBe(INCARNATION) + + // Record and pane agree on a new process, but it is not the one this request entered as. + db.db + .prepare('UPDATE dispatch_contexts SET process_incarnation = ? WHERE id = ?') + .run(REPLACEMENT, dispatchId) + paneIncarnation = REPLACEMENT + + expect(reporter.processIncarnation()).toBeNull() + }) + }) + describe('worker for another computer', () => { it('relays an escalation and a question to the Run home through its stale handle', async () => { setupFederatedWorker() @@ -178,6 +257,140 @@ describe('restart-reissued worker handle across finished and federated Dispatche expect(remoteRelayKinds()).toEqual([]) }) + describe('answer to a default (nonblocking) question', () => { + it('is consumed by the plain check its wake types with the stale env handle', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + expect(asked).toMatchObject({ pending: true }) + await importRemoteAnswer(askedId(asked)) + expect(remoteAnswerUnread()).toBe(true) + + const checked = await check() + + expect(checkedBodies(checked)).toContain('Approved') + expect(remoteAnswerUnread()).toBe(false) + }) + + it('stays unread when the pane now records a replacement incarnation', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + await importRemoteAnswer(askedId(asked)) + paneIncarnation = REPLACEMENT + + await expect(check()).rejects.toMatchObject({ code: 'dispatch_inactive' }) + expect(remoteAnswerUnread()).toBe(true) + }) + + it('stays unread when a check carries no env evidence for the stale handle', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + await importRemoteAnswer(askedId(asked)) + + await expect(check(STALE_HANDLE, ctx)).rejects.toMatchObject({ code: 'dispatch_inactive' }) + expect(remoteAnswerUnread()).toBe(true) + }) + + it('is not handed to the stale handle once another Dispatch owns the pane', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + await importRemoteAnswer(askedId(asked)) + db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded') + const next = db.createTask({ spec: 'next work in the same terminal' }) + createRootDispatch(db, next.id, REMINTED_HANDLE, WORKER_PANE, undefined, INCARNATION) + + const checked = await check().catch((error: unknown) => error) + + expect(checkedBodies(checked)).not.toContain('Approved') + expect(remoteAnswerUnread()).toBe(true) + }) + }) + + describe('after its remote Dispatch finished and local work began', () => { + function setupLocalWorkAfterRemoteHistory(handle: string): { + taskId: string + dispatchId: string + } { + setupFederatedWorker() + db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded') + const task = db.createTask({ spec: 'local work after the remote Dispatch finished' }) + const dispatch = createRootDispatch( + db, + task.id, + handle, + WORKER_PANE, + undefined, + INCARNATION + ) + db.db + .prepare('UPDATE dispatch_contexts SET host_scope = ? WHERE id = ?') + .run(HOST_SCOPE, dispatch.id) + return { taskId: task.id, dispatchId: dispatch.id } + } + + const currentEnv = (): RpcContext => ({ + ...ctx, + orchestrationCompatibilityEvidence: { + terminalHandle: REMINTED_HANDLE, + paneKey: WORKER_PANE, + launchToken: 'launch-token' + } + }) + + it.each([ + ['its stale env handle', STALE_HANDLE], + ['its current handle', REMINTED_HANDLE] + ])('settles the local Task on a default worker_done from %s', async (_label, handle) => { + const { taskId, dispatchId } = setupLocalWorkAfterRemoteHistory(handle) + + await expect( + harness.call( + 'orchestration.send', + { + from: handle, + subject: 'done', + type: 'worker_done', + payload: JSON.stringify({ taskId, dispatchId, outcome: 'succeeded' }) + }, + handle === STALE_HANDLE ? workerEnv() : currentEnv() + ) + ).resolves.toBeDefined() + + expect(db.getTask(taskId)?.status).toBe('completed') + expect(db.getDispatchContextById(dispatchId)?.status).toBe('completed') + expect(remoteRelayKinds()).toEqual([]) + }) + + it('still refuses mail that names the finished remote Dispatch', async () => { + setupLocalWorkAfterRemoteHistory(STALE_HANDLE) + + await expect( + harness.call( + 'orchestration.send', + { + from: STALE_HANDLE, + to: `dispatch:${REMOTE_DISPATCH}`, + subject: 'late', + type: 'status' + }, + workerEnv() + ) + ).rejects.toMatchObject({ code: 'run_home_unreachable' }) + }) + }) + + it('tells a finished worker its default worker_done cannot reach its coordinator', async () => { + setupFederatedWorker() + db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded') + + await expect( + send('worker_done', { + taskId: 'task_remote_restart', + dispatchId: REMOTE_DISPATCH, + outcome: 'succeeded' + }) + ).rejects.toMatchObject({ code: 'run_home_unreachable' }) + }) + it('tells a finished worker its coordinator is unreachable instead of a bare refusal', async () => { setupFederatedWorker() db.db diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts index 97281fd9378..61541ce511d 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts @@ -67,14 +67,9 @@ export const ORCHESTRATION_SEND_METHODS = [ ) => { const db = runtime.getOrchestrationDb() const from = params.from ?? 'unknown' - const attestedCaller = - orchestrationCompatibilityCallerAuthority?.terminalHandle === from - ? orchestrationCompatibilityCallerAuthority - : undefined - // Why: attested hook identity survives graph remount; caller params never supply lifecycle authority. const reporter = resolveWorkerReporterIdentity(runtime, db, { from, - attested: attestedCaller, + authority: orchestrationCompatibilityCallerAuthority, hasSessionCaller: Boolean(orchestrationCaller), evidence: orchestrationCompatibilityEvidence }) @@ -109,7 +104,7 @@ export const ORCHESTRATION_SEND_METHODS = [ }) } - refuseFinishedRemoteWorker(db, senderPaneKey, params.to) + refuseFinishedRemoteWorker(db, { address: from, paneKey: senderPaneKey }, params.to) const runGroup = params.to && isGroupAddress(params.to) &&