fix(orchestration): bind agent-status evidence on process incarnation

A hook row carries no incarnation, so a row replayed after a runtime restart was
indistinguishable from a current one by pane key and terminal handle alone, and
the reminted-pane arm accepted a dispatch-labelled row on the mere PRESENCE of a
durable `process_incarnation`. Both false binds report a dead process as `live`.

The pane's incarnation at mint time now has to equal the incarnation the durable
resource named. A worker with no materialized resource keeps binding on pane and
handle: absence there is not a contradicting authority, and fencing it out would
report a running unsupervised worker as missing.
This commit is contained in:
Jinwoo-H
2026-09-05 05:34:02 -04:00
parent f9aaade3b8
commit 97a5ecc1a5
3 changed files with 107 additions and 15 deletions
@@ -162,4 +162,64 @@ describe('local fleet liveness from a hook row that carries only a pane key', ()
reason: 'missing_status'
})
})
// A hook row carries no incarnation of its own, so a row replayed after a runtime restart
// is indistinguishable from a current one by pane and handle alone. The pane's incarnation
// at mint time is what says which process the evidence is about.
it('refuses a replayed row once the pane runs a different incarnation', () => {
const page = projectFleetWorkerPage(
createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: 'pty-fleet:inc-2' }),
createDb(),
DISPATCH_ID
)
expect(page?.workers[0]?.liveness).toMatchObject({
verdict: 'unverifiable',
reason: 'missing_status'
})
})
it('refuses a replayed row before the restarted runtime has rebound the incarnation', () => {
const page = projectFleetWorkerPage(
createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: null }),
createDb(),
DISPATCH_ID
)
expect(page?.workers[0]?.liveness).toMatchObject({
verdict: 'unverifiable',
reason: 'missing_status'
})
})
// The positive control the fail-closed tightening owes: once the rebind lands on the
// incarnation the durable resource named, the same pane reads live again.
it('reads live again once the rebind restores the durable incarnation', () => {
const page = projectFleetWorkerPage(
createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: PROCESS_INCARNATION }),
createDb(),
DISPATCH_ID
)
expect(page?.workers[0]?.liveness).toMatchObject({ verdict: 'live', source: 'agent_status' })
})
// HEAD accepted a reminted pane on `Boolean(resource.processIncarnation)` — presence, not
// equality — so a dispatch-labelled row from the previous incarnation bound to the new worker.
it('refuses a reminted pane whose dispatch matches but whose incarnation does not', () => {
const page = projectFleetWorkerPage(
createRuntime({
handleForPane: TERMINAL_HANDLE,
orchestration: { dispatchId: DISPATCH_ID } as AgentStatusOrchestrationContext,
incarnationForHandle: 'pty-fleet:inc-2'
}),
createDb(),
DISPATCH_ID
)
expect(page?.workers[0]?.liveness).toMatchObject({
verdict: 'unverifiable',
reason: 'missing_status'
})
})
})
@@ -283,11 +283,16 @@ describe('orchestration fleet projection', () => {
const result = projectOrchestrationFleet({
workers: [durable],
statuses: [
status('new', 100, {
paneKey: 'new-tab:new-leaf',
terminalHandle: 'term-worker',
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
})
status(
'new',
100,
{
paneKey: 'new-tab:new-leaf',
terminalHandle: 'term-worker',
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
},
'pty:inc-2'
)
],
now: 100
})
@@ -300,11 +305,16 @@ describe('orchestration fleet projection', () => {
projectOrchestrationFleet({
workers: [durable],
statuses: [
status('new', 100, {
paneKey: 'new-tab:new-leaf',
terminalHandle: 'term-other',
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
})
status(
'new',
100,
{
paneKey: 'new-tab:new-leaf',
terminalHandle: 'term-other',
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
},
'pty:inc-2'
)
],
now: 100
}).workers[0]?.liveness.verdict
@@ -331,11 +341,16 @@ describe('orchestration fleet projection', () => {
})
],
statuses: [
status('session-only', 100, {
providerSessionOnly: true,
orchestration: { taskId: 'task-session-only', dispatchId: 'session-only' },
providerSession: { key: 'session_id', id: 'session-1' }
})
status(
'session-only',
100,
{
providerSessionOnly: true,
orchestration: { taskId: 'task-session-only', dispatchId: 'session-only' },
providerSession: { key: 'session_id', id: 'session-1' }
},
'pty:inc-1'
)
],
now: 100
})
@@ -1,6 +1,7 @@
import {
fleetWorkerIdentity,
type FleetAgentStatusEvidence,
type FleetEvidenceBinding,
type FleetWorkerIdentity
} from './orchestration-fleet-agent-status-evidence'
import type { FleetDurableWorker } from './orchestration-fleet-projection'
@@ -120,6 +121,9 @@ function statusIdentityMatchesWorker(
if (remoteTargetId && evidence.activity.connectionId !== remoteTargetId) {
return false
}
if (!incarnationMatchesWorker(worker, binding)) {
return false
}
const paneMatches = identity.kind !== 'pane_and_terminal' || binding.paneKey === identity.paneKey
if (binding.kind === 'worker') {
// A row that names this dispatch on this handle may be a reminted pane; the durable
@@ -136,6 +140,19 @@ function statusIdentityMatchesWorker(
)
}
/** The durable resource names the incarnation the worker was dispatched onto. A hook row carries
* no incarnation of its own, so the pane's incarnation at mint time is what says which process
* the evidence describes; a row minted against a different one is evidence about that process.
* A worker with no materialized resource has no incarnation authority to contradict, and
* fencing it out on absence would report a running unsupervised worker as missing. */
function incarnationMatchesWorker(
worker: FleetDurableWorker,
binding: Exclude<FleetEvidenceBinding, { kind: 'unresolved' }>
): boolean {
const durable = worker.resource?.processIncarnation
return !durable || durable === binding.processIncarnation
}
function uniqueOwner(ownersByKey: Map<string, Set<string>>, key: string | null): boolean {
return key ? ownersByKey.get(key)?.size === 1 : true
}