diff --git a/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts b/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts index d4239ab28d1..fc791c878d2 100644 --- a/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts +++ b/src/main/runtime/orchestration/workers/reports/reminted-worker-reporter.ts @@ -153,25 +153,33 @@ export function resolveWorkerReporterIdentity( db: OrchestrationDb, args: { from: string - attested: - | Pick + /** The hook-attested caller, which speaks for `from` only when it names that same handle. */ + authority: + | Pick< + OrchestrationCompatibilityCallerAuthority, + 'terminalHandle' | 'paneKey' | 'processIncarnation' + > | undefined hasSessionCaller: boolean evidence: OrchestrationCompatibilityEvidence | undefined } ): { paneKey: string | null; processIncarnation: () => string | null } { - const { from, attested } = args + const { from } = args + // Why: attested hook identity survives graph remount; caller params never supply lifecycle authority. + const attested = args.authority?.terminalHandle === from ? args.authority : undefined const declaredPaneKey = attested?.paneKey ?? runtime.getTerminalPaneKey(from) const remint = () => args.hasSessionCaller || attested || declaredPaneKey ? null : resolveRemintedWorkerReporter(runtime, db, { from, evidence: args.evidence }) const reminted = remint() - // The re-read must still speak for the same Dispatch and pane that routing already used. + // The re-read must still speak for the same Dispatch, pane and process that routing already used, so a + // record rewritten to another incarnation after entry cannot carry the request over to it. const remintedAgain = (entry: RemintedWorkerReporter): string | null => { const again = remint() return again?.dispatchId === entry.dispatchId && - isEquivalentPaneKey(again.paneKey, entry.paneKey) + isEquivalentPaneKey(again.paneKey, entry.paneKey) && + again.processIncarnation === entry.processIncarnation ? again.processIncarnation : null } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts index 972bc41953e..af07176e9fc 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts @@ -56,7 +56,7 @@ export const ORCHESTRATION_ASK_METHODS = [ // A runtime restart can reissue a surviving worker PTY's handle; recover it by exact incarnation. const reporter = resolveWorkerReporterIdentity(runtime, db, { from, - attested: undefined, + authority: undefined, hasSessionCaller: Boolean(orchestrationCaller), evidence: orchestrationCompatibilityEvidence }) @@ -101,7 +101,7 @@ export const ORCHESTRATION_ASK_METHODS = [ 'question' ) if (!askingDispatch) { - refuseFinishedRemoteWorker(db, paneKey, undefined) + refuseFinishedRemoteWorker(db, { address: from, paneKey }, undefined) throw new OrchestrationError( 'dispatch_inactive', 'ask requires an active supervised Dispatch.' diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts index 11af8dd3c25..493039f2750 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-methods.ts @@ -10,6 +10,7 @@ import { withRunCheckNext } from '../receipt-next' import { orchestrationSkillRecoveryData } from '../../../../../../shared/orchestration/orchestration-rpc-contract' import { hasRunBindingKey } from '../../../../orchestration/addressing/orchestration-caller-identity' import { orchestrationCallerIdentity } from '../runs/run-scope' +import { resolveWorkerReporterIdentity } from '../../../../orchestration/workers/reports/reminted-worker-reporter' import { callerHoldsDispatchPane, dispatchFenced, @@ -44,11 +45,19 @@ export const ORCHESTRATION_CHECK_METHODS = [ const handle = params.terminal ?? 'unknown' const typeFilter = parseMessageTypes(params.types) + // A restart can reissue a surviving worker PTY's handle; the woken worker still checks with its env + // handle. Recover it by the Dispatch's exact recorded incarnation, as its ask and reports were. + const reporter = resolveWorkerReporterIdentity(runtime, db, { + from: handle, + authority: undefined, + hasSessionCaller: Boolean(orchestrationCaller), + evidence: orchestrationCompatibilityEvidence + }) const caller = orchestrationCallerIdentity(runtime, { handle, session: orchestrationCaller, // Why: a live runtime handle is authoritative; pane metadata is only the restart fallback. - paneKey: runtime.getTerminalPaneKey(handle) ?? params.terminalPaneKey + paneKey: reporter.paneKey ?? params.terminalPaneKey }) const paneKey = caller.paneKey ?? undefined const boundRun = hasRunBindingKey(caller) ? db.getCurrentRunForCoordinator(caller) : undefined @@ -94,7 +103,7 @@ export const ORCHESTRATION_CHECK_METHODS = [ !db.isRemoteAttachmentProcessCurrent({ dispatchId: remoteAttachment.dispatch_id, paneKey: paneKey ?? null, - processIncarnation: runtime.getTerminalProcessIncarnation(handle) + processIncarnation: reporter.processIncarnation() }) ) { throw new OrchestrationError( @@ -114,6 +123,7 @@ export const ORCHESTRATION_CHECK_METHODS = [ signal, activeDispatch, remoteAttachment, + processIncarnation: reporter.processIncarnation, recordMutationReceipt, acknowledgeOnly: !params.wait }) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts index 8ae22f5fa61..559f0d599d4 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-worker.ts @@ -28,6 +28,8 @@ export async function checkWorkerMailbox(args: { signal: AbortSignal | undefined activeDispatch: ActiveDispatch | undefined remoteAttachment: RemoteAttachment | undefined + /** The caller's process, re-read at each fence; a restart-reissued handle supplies its recovered one. */ + processIncarnation?: () => string | null wakeTypes?: MessageType[] revalidateConsumer?: () => void deferDelivery?: () => boolean @@ -100,7 +102,9 @@ export async function checkWorkerMailbox(args: { db.isRemoteAttachmentProcessCurrent({ dispatchId: current.dispatch_id, paneKey, - processIncarnation: runtime.getTerminalProcessIncarnation(handle) + processIncarnation: args.processIncarnation + ? args.processIncarnation() + : runtime.getTerminalProcessIncarnation(handle) }) ) { return diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts index 791c8a2905d..789c20e2b16 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/finished-worker-mail.ts @@ -85,17 +85,30 @@ export function finishedWorkerSend

( } } -/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. */ +/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. + * A worker since given local work is no longer that finished worker: its default mail goes to its + * working Dispatch, so only mail naming the finished remote Dispatch is refused. */ export function refuseFinishedRemoteWorker( db: OrchestrationDb, - paneKey: string | undefined, + sender: { address: string; paneKey: string | undefined }, to: string | undefined ): void { + const { address, paneKey } = sender const attachment = paneKey ? db.findLatestRemoteAttachmentForPane(paneKey) : undefined - if (attachment && (!to || to === `dispatch:${attachment.dispatch_id}`)) { - throw new OrchestrationError( - 'run_home_unreachable', - `Dispatch ${attachment.dispatch_id} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.` - ) + if (!attachment) { + return + } + if (to === `dispatch:${attachment.dispatch_id}`) { + throw finishedRemoteUnreachable(attachment.dispatch_id) + } + if (!to && !db.getActiveDispatchForIdentity(address, paneKey)) { + throw finishedRemoteUnreachable(attachment.dispatch_id) } } + +function finishedRemoteUnreachable(dispatchId: string): OrchestrationError { + return new OrchestrationError( + 'run_home_unreachable', + `Dispatch ${dispatchId} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.` + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts index d874561208c..395e611d265 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/reminted-worker-report-continuity.test.ts @@ -6,6 +6,7 @@ import type { OrchestrationDb } from '../../../../orchestration/db/orchestration import type { OrcaRuntimeService } from '../../../../orca-runtime' import { createOrchestrationRpcHarness } from '../rpc-test-harness' import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture' +import { resolveWorkerReporterIdentity } from '../../../../orchestration/workers/reports/reminted-worker-reporter' const STALE_HANDLE = 'term_exported-before-restart' const REMINTED_HANDLE = 'term_reissued-after-restart' @@ -116,6 +117,48 @@ describe('restart-reissued worker handle across finished and federated Dispatche workerEnv() ) + /** The plain `orchestration check` an Orca wake types: the env handle and pane, nothing else. */ + const check = (from: string = STALE_HANDLE, env: RpcContext = workerEnv()) => + harness.call('orchestration.check', { terminal: from, terminalPaneKey: WORKER_PANE }, env) + + const askedId = (asked: unknown): string => + asked && typeof asked === 'object' && 'messageId' in asked ? String(asked.messageId) : '' + + const checkedBodies = (checked: unknown): string[] => + checked && + typeof checked === 'object' && + 'messages' in checked && + Array.isArray(checked.messages) + ? checked.messages.map((message: { body?: unknown }) => String(message.body)) + : [] + + /** The Run home answers the worker's question; the answer lands in its Dispatch mail. */ + const importRemoteAnswer = (questionId: string) => + harness.call( + 'orchestration.federationImport', + { + dispatchId: REMOTE_DISPATCH, + items: [ + { + dispatch_id: REMOTE_DISPATCH, + direction: 'to_worker', + sequence: 1, + message_id: 'relay_answer', + kind: 'reply', + payload: JSON.stringify({ + questionId, + answerMessageId: 'msg_remote_answer', + body: 'Approved' + }) + } + ] + }, + { ...ctx, authenticatedCallerFingerprint: 'home_peer' } + ) + + const remoteAnswerUnread = () => + db.getUnreadMessages(`dispatch:${REMOTE_DISPATCH}`).some((m) => m.body === 'Approved') + const remoteRelayKinds = () => db .listFederationRelay({ dispatchId: REMOTE_DISPATCH, direction: 'to_home', afterSequence: 0 }) @@ -158,6 +201,42 @@ describe('restart-reissued worker handle across finished and federated Dispatche }) }) + describe('working local worker', () => { + it('consumes its default question answer with the plain check its wake types', async () => { + setupLocalWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + expect(asked).toMatchObject({ pending: true }) + await harness.call( + 'orchestration.reply', + { id: askedId(asked), body: 'Approved locally', from: 'term_coord' }, + ctx + ) + + expect(checkedBodies(await check())).toContain('Approved locally') + }) + }) + + describe('admission re-read', () => { + it('refuses a Dispatch record rewritten to another incarnation after entry', () => { + const { dispatchId } = setupLocalWorker() + const reporter = resolveWorkerReporterIdentity(runtime, db, { + from: STALE_HANDLE, + authority: undefined, + hasSessionCaller: false, + evidence: workerEnv().orchestrationCompatibilityEvidence + }) + expect(reporter.processIncarnation()).toBe(INCARNATION) + + // Record and pane agree on a new process, but it is not the one this request entered as. + db.db + .prepare('UPDATE dispatch_contexts SET process_incarnation = ? WHERE id = ?') + .run(REPLACEMENT, dispatchId) + paneIncarnation = REPLACEMENT + + expect(reporter.processIncarnation()).toBeNull() + }) + }) + describe('worker for another computer', () => { it('relays an escalation and a question to the Run home through its stale handle', async () => { setupFederatedWorker() @@ -178,6 +257,140 @@ describe('restart-reissued worker handle across finished and federated Dispatche expect(remoteRelayKinds()).toEqual([]) }) + describe('answer to a default (nonblocking) question', () => { + it('is consumed by the plain check its wake types with the stale env handle', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + expect(asked).toMatchObject({ pending: true }) + await importRemoteAnswer(askedId(asked)) + expect(remoteAnswerUnread()).toBe(true) + + const checked = await check() + + expect(checkedBodies(checked)).toContain('Approved') + expect(remoteAnswerUnread()).toBe(false) + }) + + it('stays unread when the pane now records a replacement incarnation', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + await importRemoteAnswer(askedId(asked)) + paneIncarnation = REPLACEMENT + + await expect(check()).rejects.toMatchObject({ code: 'dispatch_inactive' }) + expect(remoteAnswerUnread()).toBe(true) + }) + + it('stays unread when a check carries no env evidence for the stale handle', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + await importRemoteAnswer(askedId(asked)) + + await expect(check(STALE_HANDLE, ctx)).rejects.toMatchObject({ code: 'dispatch_inactive' }) + expect(remoteAnswerUnread()).toBe(true) + }) + + it('is not handed to the stale handle once another Dispatch owns the pane', async () => { + setupFederatedWorker() + const asked = await ask({ question: 'Proceed?', wait: false }) + await importRemoteAnswer(askedId(asked)) + db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded') + const next = db.createTask({ spec: 'next work in the same terminal' }) + createRootDispatch(db, next.id, REMINTED_HANDLE, WORKER_PANE, undefined, INCARNATION) + + const checked = await check().catch((error: unknown) => error) + + expect(checkedBodies(checked)).not.toContain('Approved') + expect(remoteAnswerUnread()).toBe(true) + }) + }) + + describe('after its remote Dispatch finished and local work began', () => { + function setupLocalWorkAfterRemoteHistory(handle: string): { + taskId: string + dispatchId: string + } { + setupFederatedWorker() + db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded') + const task = db.createTask({ spec: 'local work after the remote Dispatch finished' }) + const dispatch = createRootDispatch( + db, + task.id, + handle, + WORKER_PANE, + undefined, + INCARNATION + ) + db.db + .prepare('UPDATE dispatch_contexts SET host_scope = ? WHERE id = ?') + .run(HOST_SCOPE, dispatch.id) + return { taskId: task.id, dispatchId: dispatch.id } + } + + const currentEnv = (): RpcContext => ({ + ...ctx, + orchestrationCompatibilityEvidence: { + terminalHandle: REMINTED_HANDLE, + paneKey: WORKER_PANE, + launchToken: 'launch-token' + } + }) + + it.each([ + ['its stale env handle', STALE_HANDLE], + ['its current handle', REMINTED_HANDLE] + ])('settles the local Task on a default worker_done from %s', async (_label, handle) => { + const { taskId, dispatchId } = setupLocalWorkAfterRemoteHistory(handle) + + await expect( + harness.call( + 'orchestration.send', + { + from: handle, + subject: 'done', + type: 'worker_done', + payload: JSON.stringify({ taskId, dispatchId, outcome: 'succeeded' }) + }, + handle === STALE_HANDLE ? workerEnv() : currentEnv() + ) + ).resolves.toBeDefined() + + expect(db.getTask(taskId)?.status).toBe('completed') + expect(db.getDispatchContextById(dispatchId)?.status).toBe('completed') + expect(remoteRelayKinds()).toEqual([]) + }) + + it('still refuses mail that names the finished remote Dispatch', async () => { + setupLocalWorkAfterRemoteHistory(STALE_HANDLE) + + await expect( + harness.call( + 'orchestration.send', + { + from: STALE_HANDLE, + to: `dispatch:${REMOTE_DISPATCH}`, + subject: 'late', + type: 'status' + }, + workerEnv() + ) + ).rejects.toMatchObject({ code: 'run_home_unreachable' }) + }) + }) + + it('tells a finished worker its default worker_done cannot reach its coordinator', async () => { + setupFederatedWorker() + db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded') + + await expect( + send('worker_done', { + taskId: 'task_remote_restart', + dispatchId: REMOTE_DISPATCH, + outcome: 'succeeded' + }) + ).rejects.toMatchObject({ code: 'run_home_unreachable' }) + }) + it('tells a finished worker its coordinator is unreachable instead of a bare refusal', async () => { setupFederatedWorker() db.db diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts index 97281fd9378..61541ce511d 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts @@ -67,14 +67,9 @@ export const ORCHESTRATION_SEND_METHODS = [ ) => { const db = runtime.getOrchestrationDb() const from = params.from ?? 'unknown' - const attestedCaller = - orchestrationCompatibilityCallerAuthority?.terminalHandle === from - ? orchestrationCompatibilityCallerAuthority - : undefined - // Why: attested hook identity survives graph remount; caller params never supply lifecycle authority. const reporter = resolveWorkerReporterIdentity(runtime, db, { from, - attested: attestedCaller, + authority: orchestrationCompatibilityCallerAuthority, hasSessionCaller: Boolean(orchestrationCaller), evidence: orchestrationCompatibilityEvidence }) @@ -109,7 +104,7 @@ export const ORCHESTRATION_SEND_METHODS = [ }) } - refuseFinishedRemoteWorker(db, senderPaneKey, params.to) + refuseFinishedRemoteWorker(db, { address: from, paneKey: senderPaneKey }, params.to) const runGroup = params.to && isGroupAddress(params.to) &&