mirror of
https://github.com/stablyai/orca.git
synced 2026-09-29 08:03:20 +00:00
fix(orchestration): bind agent-status evidence on process incarnation
A hook row carries no incarnation, so a row replayed after a runtime restart was indistinguishable from a current one by pane key and terminal handle alone, and the reminted-pane arm accepted a dispatch-labelled row on the mere PRESENCE of a durable `process_incarnation`. Both false binds report a dead process as `live`. The pane's incarnation at mint time now has to equal the incarnation the durable resource named. A worker with no materialized resource keeps binding on pane and handle: absence there is not a contradicting authority, and fencing it out would report a running unsupervised worker as missing.
This commit is contained in:
+60
@@ -162,4 +162,64 @@ describe('local fleet liveness from a hook row that carries only a pane key', ()
|
||||
reason: 'missing_status'
|
||||
})
|
||||
})
|
||||
|
||||
// A hook row carries no incarnation of its own, so a row replayed after a runtime restart
|
||||
// is indistinguishable from a current one by pane and handle alone. The pane's incarnation
|
||||
// at mint time is what says which process the evidence is about.
|
||||
it('refuses a replayed row once the pane runs a different incarnation', () => {
|
||||
const page = projectFleetWorkerPage(
|
||||
createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: 'pty-fleet:inc-2' }),
|
||||
createDb(),
|
||||
DISPATCH_ID
|
||||
)
|
||||
|
||||
expect(page?.workers[0]?.liveness).toMatchObject({
|
||||
verdict: 'unverifiable',
|
||||
reason: 'missing_status'
|
||||
})
|
||||
})
|
||||
|
||||
it('refuses a replayed row before the restarted runtime has rebound the incarnation', () => {
|
||||
const page = projectFleetWorkerPage(
|
||||
createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: null }),
|
||||
createDb(),
|
||||
DISPATCH_ID
|
||||
)
|
||||
|
||||
expect(page?.workers[0]?.liveness).toMatchObject({
|
||||
verdict: 'unverifiable',
|
||||
reason: 'missing_status'
|
||||
})
|
||||
})
|
||||
|
||||
// The positive control the fail-closed tightening owes: once the rebind lands on the
|
||||
// incarnation the durable resource named, the same pane reads live again.
|
||||
it('reads live again once the rebind restores the durable incarnation', () => {
|
||||
const page = projectFleetWorkerPage(
|
||||
createRuntime({ handleForPane: TERMINAL_HANDLE, incarnationForHandle: PROCESS_INCARNATION }),
|
||||
createDb(),
|
||||
DISPATCH_ID
|
||||
)
|
||||
|
||||
expect(page?.workers[0]?.liveness).toMatchObject({ verdict: 'live', source: 'agent_status' })
|
||||
})
|
||||
|
||||
// HEAD accepted a reminted pane on `Boolean(resource.processIncarnation)` — presence, not
|
||||
// equality — so a dispatch-labelled row from the previous incarnation bound to the new worker.
|
||||
it('refuses a reminted pane whose dispatch matches but whose incarnation does not', () => {
|
||||
const page = projectFleetWorkerPage(
|
||||
createRuntime({
|
||||
handleForPane: TERMINAL_HANDLE,
|
||||
orchestration: { dispatchId: DISPATCH_ID } as AgentStatusOrchestrationContext,
|
||||
incarnationForHandle: 'pty-fleet:inc-2'
|
||||
}),
|
||||
createDb(),
|
||||
DISPATCH_ID
|
||||
)
|
||||
|
||||
expect(page?.workers[0]?.liveness).toMatchObject({
|
||||
verdict: 'unverifiable',
|
||||
reason: 'missing_status'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -283,11 +283,16 @@ describe('orchestration fleet projection', () => {
|
||||
const result = projectOrchestrationFleet({
|
||||
workers: [durable],
|
||||
statuses: [
|
||||
status('new', 100, {
|
||||
paneKey: 'new-tab:new-leaf',
|
||||
terminalHandle: 'term-worker',
|
||||
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
|
||||
})
|
||||
status(
|
||||
'new',
|
||||
100,
|
||||
{
|
||||
paneKey: 'new-tab:new-leaf',
|
||||
terminalHandle: 'term-worker',
|
||||
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
|
||||
},
|
||||
'pty:inc-2'
|
||||
)
|
||||
],
|
||||
now: 100
|
||||
})
|
||||
@@ -300,11 +305,16 @@ describe('orchestration fleet projection', () => {
|
||||
projectOrchestrationFleet({
|
||||
workers: [durable],
|
||||
statuses: [
|
||||
status('new', 100, {
|
||||
paneKey: 'new-tab:new-leaf',
|
||||
terminalHandle: 'term-other',
|
||||
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
|
||||
})
|
||||
status(
|
||||
'new',
|
||||
100,
|
||||
{
|
||||
paneKey: 'new-tab:new-leaf',
|
||||
terminalHandle: 'term-other',
|
||||
orchestration: { taskId: 'task-dispatch-1', dispatchId: 'dispatch-1' }
|
||||
},
|
||||
'pty:inc-2'
|
||||
)
|
||||
],
|
||||
now: 100
|
||||
}).workers[0]?.liveness.verdict
|
||||
@@ -331,11 +341,16 @@ describe('orchestration fleet projection', () => {
|
||||
})
|
||||
],
|
||||
statuses: [
|
||||
status('session-only', 100, {
|
||||
providerSessionOnly: true,
|
||||
orchestration: { taskId: 'task-session-only', dispatchId: 'session-only' },
|
||||
providerSession: { key: 'session_id', id: 'session-1' }
|
||||
})
|
||||
status(
|
||||
'session-only',
|
||||
100,
|
||||
{
|
||||
providerSessionOnly: true,
|
||||
orchestration: { taskId: 'task-session-only', dispatchId: 'session-only' },
|
||||
providerSession: { key: 'session_id', id: 'session-1' }
|
||||
},
|
||||
'pty:inc-1'
|
||||
)
|
||||
],
|
||||
now: 100
|
||||
})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
fleetWorkerIdentity,
|
||||
type FleetAgentStatusEvidence,
|
||||
type FleetEvidenceBinding,
|
||||
type FleetWorkerIdentity
|
||||
} from './orchestration-fleet-agent-status-evidence'
|
||||
import type { FleetDurableWorker } from './orchestration-fleet-projection'
|
||||
@@ -120,6 +121,9 @@ function statusIdentityMatchesWorker(
|
||||
if (remoteTargetId && evidence.activity.connectionId !== remoteTargetId) {
|
||||
return false
|
||||
}
|
||||
if (!incarnationMatchesWorker(worker, binding)) {
|
||||
return false
|
||||
}
|
||||
const paneMatches = identity.kind !== 'pane_and_terminal' || binding.paneKey === identity.paneKey
|
||||
if (binding.kind === 'worker') {
|
||||
// A row that names this dispatch on this handle may be a reminted pane; the durable
|
||||
@@ -136,6 +140,19 @@ function statusIdentityMatchesWorker(
|
||||
)
|
||||
}
|
||||
|
||||
/** The durable resource names the incarnation the worker was dispatched onto. A hook row carries
|
||||
* no incarnation of its own, so the pane's incarnation at mint time is what says which process
|
||||
* the evidence describes; a row minted against a different one is evidence about that process.
|
||||
* A worker with no materialized resource has no incarnation authority to contradict, and
|
||||
* fencing it out on absence would report a running unsupervised worker as missing. */
|
||||
function incarnationMatchesWorker(
|
||||
worker: FleetDurableWorker,
|
||||
binding: Exclude<FleetEvidenceBinding, { kind: 'unresolved' }>
|
||||
): boolean {
|
||||
const durable = worker.resource?.processIncarnation
|
||||
return !durable || durable === binding.processIncarnation
|
||||
}
|
||||
|
||||
function uniqueOwner(ownersByKey: Map<string, Set<string>>, key: string | null): boolean {
|
||||
return key ? ownersByKey.get(key)?.size === 1 : true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user