From 97a5ecc1a59bfee3f3f1bb1669fc4a763e578a2d Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Sat, 5 Sep 2026 05:34:02 -0400 Subject: [PATCH] fix(orchestration): bind agent-status evidence on process incarnation A hook row carries no incarnation, so a row replayed after a runtime restart was indistinguishable from a current one by pane key and terminal handle alone, and the reminted-pane arm accepted a dispatch-labelled row on the mere PRESENCE of a durable `process_incarnation`. Both false binds report a dead process as `live`. The pane's incarnation at mint time now has to equal the incarnation the durable resource named. A worker with no materialized resource keeps binding on pane and handle: absence there is not a contradicting authority, and fencing it out would report a running unsupervised worker as missing. --- .../fleet-status-terminal-identity.test.ts | 60 +++++++++++++++++++ .../orchestration-fleet-projection.test.ts | 45 +++++++++----- .../orchestration-fleet-status-index.ts | 17 ++++++ 3 files changed, 107 insertions(+), 15 deletions(-) diff --git a/src/main/runtime/rpc/methods/orchestration/worker/fleet-status-terminal-identity.test.ts b/src/main/runtime/rpc/methods/orchestration/worker/fleet-status-terminal-identity.test.ts index 8f7fcefc22a..f350ae1cf0f 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/fleet-status-terminal-identity.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/fleet-status-terminal-identity.test.ts @@ -162,4 +162,64 @@ describe('local fleet liveness from a hook row that carries only a pane key', () reason: 'missing_status' }) }) + + // A hook row carries no incarnation of its own, so a row replayed after a runtime restart + // is indistinguishable from a current one by pane and handle alone. The pane's incarnation + // at mint time is what says which process the evidence is about. + it('refuses a replayed row once the pane runs a different incarnation', () => { + const page = projectFleetWorkerPage( + createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: 'pty-fleet:inc-2' }), + createDb(), + DISPATCH_ID + ) + + expect(page?.workers[0]?.liveness).toMatchObject({ + verdict: 'unverifiable', + reason: 'missing_status' + }) + }) + + it('refuses a replayed row before the restarted runtime has rebound the incarnation', () => { + const page = projectFleetWorkerPage( + createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: null }), + createDb(), + DISPATCH_ID + ) + + expect(page?.workers[0]?.liveness).toMatchObject({ + verdict: 'unverifiable', + reason: 'missing_status' + }) + }) + + // The positive control the fail-closed tightening owes: once the rebind lands on the + // incarnation the durable resource named, the same pane reads live again. + it('reads live again once the rebind restores the durable incarnation', () => { + const page = projectFleetWorkerPage( + createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: PROCESS_INCARNATION }), + createDb(), + DISPATCH_ID + ) + + expect(page?.workers[0]?.liveness).toMatchObject({ verdict: 'live', source: 'agent_status' }) + }) + + // HEAD accepted a reminted pane on `Boolean(resource.processIncarnation)` — presence, not + // equality — so a dispatch-labelled row from the previous incarnation bound to the new worker. + it('refuses a reminted pane whose dispatch matches but whose incarnation does not', () => { + const page = projectFleetWorkerPage( + createRuntime({ + handleForPane: TERMINAL_HANDLE, + orchestration: { dispatchId: DISPATCH_ID } as AgentStatusOrchestrationContext, + incarnationForHandle: 'pty-fleet:inc-2' + }), + createDb(), + DISPATCH_ID + ) + + expect(page?.workers[0]?.liveness).toMatchObject({ + verdict: 'unverifiable', + reason: 'missing_status' + }) + }) }) diff --git a/src/shared/orchestration-fleet-projection.test.ts b/src/shared/orchestration-fleet-projection.test.ts index d0858ebb24e..f8cc10de176 100644 --- a/src/shared/orchestration-fleet-projection.test.ts +++ b/src/shared/orchestration-fleet-projection.test.ts @@ -283,11 +283,16 @@ describe('orchestration fleet projection', () => { const result = projectOrchestrationFleet({ workers: [durable], statuses: [ - status('new', 100, { - paneKey: 'new-tab:new-leaf', - terminalHandle: 'term-worker', - orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' } - }) + status( + 'new', + 100, + { + paneKey: 'new-tab:new-leaf', + terminalHandle: 'term-worker', + orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' } + }, + 'pty:inc-2' + ) ], now: 100 }) @@ -300,11 +305,16 @@ describe('orchestration fleet projection', () => { projectOrchestrationFleet({ workers: [durable], statuses: [ - status('new', 100, { - paneKey: 'new-tab:new-leaf', - terminalHandle: 'term-other', - orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' } - }) + status( + 'new', + 100, + { + paneKey: 'new-tab:new-leaf', + terminalHandle: 'term-other', + orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' } + }, + 'pty:inc-2' + ) ], now: 100 }).workers[0]?.liveness.verdict @@ -331,11 +341,16 @@ describe('orchestration fleet projection', () => { }) ], statuses: [ - status('session-only', 100, { - providerSessionOnly: true, - orchestration: { taskId: 'task-session-only', dispatchId: 'session-only' }, - providerSession: { key: 'session_id', id: 'session-1' } - }) + status( + 'session-only', + 100, + { + providerSessionOnly: true, + orchestration: { taskId: 'task-session-only', dispatchId: 'session-only' }, + providerSession: { key: 'session_id', id: 'session-1' } + }, + 'pty:inc-1' + ) ], now: 100 }) diff --git a/src/shared/orchestration-fleet-status-index.ts b/src/shared/orchestration-fleet-status-index.ts index 5574f7733ad..1c5a9d63b38 100644 --- a/src/shared/orchestration-fleet-status-index.ts +++ b/src/shared/orchestration-fleet-status-index.ts @@ -1,6 +1,7 @@ import { fleetWorkerIdentity, type FleetAgentStatusEvidence, + type FleetEvidenceBinding, type FleetWorkerIdentity } from './orchestration-fleet-agent-status-evidence' import type { FleetDurableWorker } from './orchestration-fleet-projection' @@ -120,6 +121,9 @@ function statusIdentityMatchesWorker( if (remoteTargetId && evidence.activity.connectionId !== remoteTargetId) { return false } + if (!incarnationMatchesWorker(worker, binding)) { + return false + } const paneMatches = identity.kind !== 'pane_and_terminal' || binding.paneKey === identity.paneKey if (binding.kind === 'worker') { // A row that names this dispatch on this handle may be a reminted pane; the durable @@ -136,6 +140,19 @@ function statusIdentityMatchesWorker( ) } +/** The durable resource names the incarnation the worker was dispatched onto. A hook row carries + * no incarnation of its own, so the pane's incarnation at mint time is what says which process + * the evidence describes; a row minted against a different one is evidence about that process. + * A worker with no materialized resource has no incarnation authority to contradict, and + * fencing it out on absence would report a running unsupervised worker as missing. */ +function incarnationMatchesWorker( + worker: FleetDurableWorker, + binding: Exclude +): boolean { + const durable = worker.resource?.processIncarnation + return !durable || durable === binding.processIncarnation +} + function uniqueOwner(ownersByKey: Map>, key: string | null): boolean { return key ? ownersByKey.get(key)?.size === 1 : true }