From 9bb75fdf57ce1d78e5a7ebce2fa7076d8d23944f Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 9 Nov 2026 02:31:31 -0500 Subject: [PATCH] fix(orchestration): fence owed-stop resends per cycle and record a server's unsettled stop as unknown --- .../runtime-orchestration-federation.ts | 4 +- .../federation/federated-worker-stop.ts | 93 +++++++++++++------ ...tion-home-record-scenarios.test-support.ts | 27 ++++++ 3 files changed, 93 insertions(+), 31 deletions(-) diff --git a/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts b/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts index 80741b6bd90..dbcfda4a716 100644 --- a/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts +++ b/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts @@ -217,6 +217,7 @@ export class RuntimeOrchestrationFederation { return } const runtime = this.runtime + const generation = this.relayGeneration void resendOwedFederatedStop( { getOrchestrationDb: () => runtime.getOrchestrationDb(), @@ -225,7 +226,8 @@ export class RuntimeOrchestrationFederation { callOrchestrationWorkerServer: (...args) => runtime.callOrchestrationWorkerServer(...args), syncOrchestrationFederatedDispatchAfterCurrent: (id) => this.syncDispatchAfterCurrent(id) }, - dispatchId + dispatchId, + () => generation === this.relayGeneration ).catch((error: unknown) => { console.warn(`[orchestration] Resending the stop for ${dispatchId} failed:`, error) }) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop.ts index 8d99a4883f5..85c7809a976 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop.ts @@ -36,10 +36,20 @@ function owedStopReason(serverName: string): string { return `${serverName} did not answer, so Orca cannot tell whether the stop reached the worker. Orca asks again when ${serverName} is back; worker-show shows when it lands.` } -/** A resend keeps one request across attempts, so a stop the server applied but whose answer was lost is - * replayed by the server rather than run again. */ -function owedStopRequestId(dispatchId: string): string { - return `owed-stop:${dispatchId}` +/** One request per owed cycle: a resend the server applied but whose answer was lost is replayed by the + * server, not run again. A later explicit stop that goes unanswered starts a new cycle, and a new request. */ +function owedStopRequestId(dispatchId: string, owedSince: string): string { + return `owed-stop:${dispatchId}:${owedSince}` +} + +/** This runtime replaced its database or relay mid-stop: nothing more is sent or written. */ +class StopFenceError extends OrchestrationError { + constructor(dispatchId: string) { + super( + 'operation_unknown', + `The orchestration database changed while Dispatch ${dispatchId} was stopping.` + ) + } } // One stop at a time per Dispatch, manual or resent: a manual stop waits out a resend in flight. @@ -85,7 +95,11 @@ export async function stopFederatedWorker( const server = resolvePinnedFederatedServer(runtime, federated) // Without the host's answer nothing is stopped: the user may have kept this worker open there. await refuseFederatedStopWhileKeptOpen(runtime, federated) + const current = () => runtime.getOrchestrationDb() === db return inStopFlight(db, dispatchId, async () => { + if (!current()) { + throw new StopFenceError(dispatchId) + } // A stop already recorded as unknown is retried as it stands: it is never begun twice. if (db.getWorkerDispatch(dispatchId)?.state !== 'stop_unknown') { const begun = db.beginWorkerStop(dispatchId, runtime.getRuntimeId()) @@ -95,20 +109,27 @@ export async function stopFederatedWorker( server, dispatchId, begun.worker.state, - orchestrationMutation.requestId + orchestrationMutation.requestId, + current ) } } - return sendFederatedStop(runtime, db, server, dispatchId, orchestrationMutation.requestId) + return sendFederatedStop(runtime, db, server, dispatchId, { + requestId: orchestrationMutation.requestId, + current + }) }) } /** The relay calls this once the server answers again; only a stop the server never answered is sent. */ export async function resendOwedFederatedStop( runtime: Omit, - dispatchId: string + dispatchId: string, + /** The relay generation that asked; a stopped or reset relay sends nothing more. */ + relayCurrent: () => boolean ): Promise { const db = runtime.getOrchestrationDb() + const current = () => relayCurrent() && runtime.getOrchestrationDb() === db if (stopFlights.get(db)?.has(dispatchId) || !isStopOwed(db, dispatchId)) { return } @@ -121,11 +142,7 @@ export async function resendOwedFederatedStop( try { await refuseFederatedStopWhileKeptOpen(runtime, federated) } catch (error) { - if ( - !isServerUnreachable(error) && - runtime.getOrchestrationDb() === db && - isStopOwed(db, dispatchId) - ) { + if (!isServerUnreachable(error) && current() && isStopOwed(db, dispatchId)) { // Kept open on the server since: the stop is not sent, and it is no longer owed. db.markWorkerStopUnknown( dispatchId, @@ -136,8 +153,13 @@ export async function resendOwedFederatedStop( return } // Re-read after the wait: a manual stop or a profile switch may have settled or replaced it. - if (runtime.getOrchestrationDb() === db && isStopOwed(db, dispatchId)) { - await sendFederatedStop(runtime, db, server, dispatchId, owedStopRequestId(dispatchId)) + const owed = db.getWorkerDispatch(dispatchId) + if (current() && owed && isStopOwed(db, dispatchId)) { + await sendFederatedStop(runtime, db, server, dispatchId, { + requestId: owedStopRequestId(dispatchId, owed.updated_at), + current, + resend: true + }) } }) } @@ -152,26 +174,30 @@ async function sendFederatedStop( db: OrchestrationDb, server: ReturnType, dispatchId: string, - requestId: string + attempt: { + requestId: string + /** False once this runtime replaced its database or relay: nothing more is sent or written. */ + current: () => boolean + /** A relay resend of an owed stop: unanswered again, it stays in the same owed cycle. */ + resend?: boolean + } ): Promise { const call = federatedStopCall(runtime, server) - // Every write after a wait goes to the database the stop began in, and only while it is still current. const fenced = (write: () => T): T => { - if (runtime.getOrchestrationDb() !== db) { - throw new OrchestrationError( - 'operation_unknown', - `The orchestration database changed while Dispatch ${dispatchId} was stopping.` - ) + if (!attempt.current()) { + throw new StopFenceError(dispatchId) } return write() } - // A concurrent path may have settled it meanwhile; that settlement stands. + // A concurrent path may have settled it meanwhile, and a resend unanswered again keeps its cycle. const stillUnknown = (reason: string, stage?: 'stop_owed' | 'stop_outcome_unknown') => - fenced(() => - db.getWorkerDispatch(dispatchId)?.state === 'stopped' - ? (db.getWorkerDispatch(dispatchId) as { state: string; last_error: string | null }) + fenced(() => { + const now = db.getWorkerDispatch(dispatchId) + return now?.state === 'stopped' || + (attempt.resend && stage === 'stop_owed' && now?.stage === 'stop_owed') + ? (now as { state: string; last_error: string | null }) : db.markWorkerStopUnknown(dispatchId, reason, stage) - ) + }) try { const status = RemoteCapabilities.parse(await call('status.get')) if (!status.capabilities?.includes(ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY)) { @@ -181,8 +207,10 @@ async function sendFederatedStop( 'none' ) } + // Checked before the only call with an effect on the server. + fenced(() => undefined) const remote = RemoteStopReceipt.parse( - await call('orchestration.federationStop', { dispatchId }, requestId) + await call('orchestration.federationStop', { dispatchId }, attempt.requestId) ) if (remote.state === 'stopped') { const worker = fenced(() => db.reconcileFederatedWorkerStop(dispatchId)) @@ -217,10 +245,11 @@ async function sendFederatedStop( ...remoteHelpers(remote, status, server.name) } } catch (error) { - if (error instanceof OrchestrationError && error.code === 'operation_unknown') { + if (error instanceof StopFenceError) { throw error } - // No answer leaves the stop owed and asked again; an answer Orca could not read leaves it unknown. + // No answer leaves the stop owed and asked again; any other answer, the server's own + // operation_unknown included, leaves it unknown for an explicit retry. const unanswered = isServerUnreachable(error) const reason = unanswered ? owedStopReason(server.name) @@ -241,7 +270,8 @@ async function stopSettledFederatedHelpers( server: ReturnType, dispatchId: string, state: string, - requestId: string + requestId: string, + current: () => boolean ): Promise { const settled = settledReceipt(dispatchId, state) const call = federatedStopCall(runtime, server) @@ -250,6 +280,9 @@ async function stopSettledFederatedHelpers( if (!status.capabilities?.includes(ORCHESTRATION_WORKER_STOP_HELPERS_RUNTIME_CAPABILITY)) { return settled } + if (!current()) { + throw new StopFenceError(dispatchId) + } const remote = await call('orchestration.federationStop', { dispatchId }, requestId) return { ...settled, ...remoteHelpers(RemoteStopReceipt.parse(remote), status, server.name) } } catch (error) { diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-home-record-scenarios.test-support.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-home-record-scenarios.test-support.ts index edf0bb3157a..ad25fe5955b 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-home-record-scenarios.test-support.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-home-record-scenarios.test-support.ts @@ -114,4 +114,31 @@ export function registerHomeRecordScenarios(harness: AutoReleaseScenarioHarness) ).resolves.toMatchObject({ ok: true, result: { state: 'stopped' } }) expect(harness.workerDb().getRemoteDispatchAttachment(dispatchId)?.state).toBe('stopped') }) + + it('records a stop the server could not settle as unknown, so it can be asked again', async () => { + const dispatchId = await harness.startRemoteWorker() + const call = harness.homeRuntime().callOrchestrationWorkerServer.bind(harness.homeRuntime()) + const pending = vi + .spyOn(harness.homeRuntime(), 'callOrchestrationWorkerServer') + .mockImplementation((environmentId, method, ...rest) => + method === 'orchestration.federationStop' + ? Promise.reject( + new OrchestrationError('operation_unknown', 'The stop is still pending.') + ) + : call(environmentId, method, ...rest) + ) + + await expect( + fromHome('orchestration.workerStop', { dispatch: dispatchId }, 'stop_pending') + ).resolves.toMatchObject({ ok: true, result: { state: 'stop_unknown' } }) + expect(harness.homeDb().getWorkerDispatch(dispatchId)).toMatchObject({ + state: 'stop_unknown', + stage: 'stop_outcome_unknown' + }) + pending.mockRestore() + + await expect( + fromHome('orchestration.workerStop', { dispatch: dispatchId }, 'stop_after_pending') + ).resolves.toMatchObject({ ok: true, result: { state: 'stopped' } }) + }) }