fix(runtime): bound restored agent authority recovery

This commit is contained in:
Jinwoo-H
2026-08-23 21:45:28 -04:00
parent bd92d61649
commit 125784bcea
8 changed files with 753 additions and 190 deletions
+18 -7
View File
@@ -10272,7 +10272,7 @@
"providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["local", "daemon", "ssh"],
"coverageNotes": "A deterministic service-state-machine oracle now models a current-contract worker and coordinator whose renderer graph identities disappear across an app/runtime update. It exercises the production verifier with restored PTY and hydrated hook commitments, proves authenticated completion replay across a fresh runtime, explicit takeover, ordinary mail routing, remote-attachment process fencing, retained Task/Dispatch/terminal identity, and unchanged fixture marker bytes, and rejects foreign pane evidence. Worktree inventory coverage now capability-gates restored presence, resolves each exact PTY on the execution host as live, unverifiable, or exited, keeps live/unverifiable identities visible without reviving stale activity, preserves byte-compatible legacy suppression, and keeps completed shell-only rows retired. Other deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, creator pane/process/Run-generation fencing, indexed retained-Run lookup scaling, the SSH in-process CLI fallback, exact legacy Task/Dispatch/report-receipt verification, JSON rejection envelopes, and capability-negotiated Run-home settlement. The federation oracle drops the first terminal ACK, restarts Run-home twice, verifies replay through the existing polling path and the persisted acknowledgment watermark, races identical reports, accepts preexisting same-outcome ACK batches, rejects conflicting batches, and covers protocol v1/v2 completion across independently updated peers. A migration unit upgrades a v26 database with a zeroed watermark without replaying older migrations. Three isolated macOS Electron journeys launch fake Codex workers through the real RPC path. Distinct installed A/B artifacts plus live SSH, WSL, folder, Linux, and Windows cutover journeys remain explicit gaps.",
"coverageNotes": "A deterministic service-state-machine oracle now models a current-contract worker and coordinator whose renderer graph identities disappear across an app/runtime update. It exercises the production verifier with restored PTY and hydrated hook commitments, proves authenticated completion replay across a fresh runtime, explicit takeover, ordinary mail routing, remote-attachment process fencing, retained Task/Dispatch/terminal identity, and unchanged fixture marker bytes, and rejects foreign pane evidence. Worktree inventory coverage now capability-gates restored presence, resolves each exact PTY on the execution host as live, unverifiable, or exited, keeps live/unverifiable identities visible without reviving stale activity, preserves byte-compatible legacy suppression, and keeps completed shell-only rows retired. Persisted surface identity must seed first authority, request-wide indexes preserve ambiguous-binding refusal without per-row PTY scans, and shared inspection admits at most eight unresolved provider probes while restored-only cache entries are pruned when hooks disappear. Other deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, creator pane/process/Run-generation fencing, indexed retained-Run lookup scaling, the SSH in-process CLI fallback, exact legacy Task/Dispatch/report-receipt verification, JSON rejection envelopes, and capability-negotiated Run-home settlement. The federation oracle drops the first terminal ACK, restarts Run-home twice, verifies replay through the existing polling path and the persisted acknowledgment watermark, races identical reports, accepts preexisting same-outcome ACK batches, rejects conflicting batches, and covers protocol v1/v2 completion across independently updated peers. A migration unit upgrades a v26 database with a zeroed watermark without replaying older migrations. Three isolated macOS Electron journeys launch fake Codex workers through the real RPC path. Distinct installed A/B artifacts plus live SSH, WSL, folder, Linux, and Windows cutover journeys remain explicit gaps.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/11107#discussion_r3663321387",
"https://github.com/stablyai/orca/issues/13858",
@@ -10355,11 +10355,21 @@
"file": "src/main/runtime/restored-agent-authority-resolver.test.ts",
"assertions": [
"one restored hook cannot borrow a replacement PTY incarnation or another execution host",
"current-only surface identity cannot establish first authority; exact persisted identity seeds it before matching current state may continue",
"conflicting current and persisted bindings remain unverifiable",
"incarnation identity survives local lifecycle-generation changes",
"tagged fallback generations cannot collide with generation-shaped incarnation ids, and optional handles do not redefine process equality",
"retired hook commitments are released without unbounded growth"
]
},
{
"file": "src/main/runtime/pty-process-liveness-broker.test.ts",
"assertions": [
"cross-PTY hanging probes remain below the broker-wide admission cap",
"retired restored-agent entries are pruned without deleting evidence shared by other process consumers",
"per-identity timeouts, settlement, TTL reuse, backoff, exact absence, and owning-inventory reconciliation remain single-flight"
]
},
{
"file": "src/main/runtime/orca-runtime.test.ts",
"assertions": [
@@ -10374,7 +10384,8 @@
"terminal_gone is the only inspection rejection that retires the exact restored row",
"fresh exact inventory cannot be overwritten by older cached terminal_gone evidence after later contact loss",
"slow foreground inspection cannot manufacture agent exit or certify a replacement incarnation",
"sequential inventory clients reuse bounded process evidence instead of polling each restored row"
"sequential inventory clients reuse bounded process evidence instead of polling each restored row",
"multi-row catalogs build two host-session authority indexes per request and inspect at most four restored rows concurrently"
]
},
{
@@ -10515,8 +10526,8 @@
"platform": "macos",
"command": "pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1",
"result": "passed",
"durationSeconds": 49.8,
"summary": "Both legacy- and current-contract isolated restart journeys kept the original live PID/PTTY/incarnation, visible inactive tab, writable ACK output, and active Dispatch, while capable worktree.ps retained the exact agent row as unverifiable because the fresh desktop could not inspect the daemon-owned process. Disabling only restored-presence negotiation made the byte-identical journey fail with agent undefined while all preceding liveness, tab, and Dispatch checks stayed green."
"durationSeconds": 54.8,
"summary": "Both legacy- and current-contract isolated restart journeys kept the original live PID/PTTY/incarnation, visible inactive tab exactly once, writable ACK output, and active Dispatch, while capable worktree.ps retained the exact agent row as unverifiable because the fresh desktop could not inspect the daemon-owned process. Disabling only restored-presence negotiation made the byte-identical journey fail with agent undefined while all preceding liveness, tab, and Dispatch checks stayed green."
},
{
"date": "2026-08-23",
@@ -10524,8 +10535,8 @@
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts --fileParallelism=false --maxWorkers=1 src/main/runtime/restored-agent-authority-resolver.test.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts",
"result": "passed",
"durationSeconds": 49.44,
"summary": "Ten focused runtime and wire files passed 1,292 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, three-poll fresh-presence reconciliation, cold handle-qualified contact loss, incarnation/host/hook replacement fencing, slow foreground completion, callback isolation, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, encrypted desktop/CLI capability advertisement, and cross-version terminal wire compatibility. Full mobile typecheck and 3,674 tests passed separately."
"durationSeconds": 27.06,
"summary": "Ten focused runtime and wire files passed 1,301 tests with one skip, including persisted-seeded first authority, replacement-current refusal, tagged incarnation identity, handle-remint continuity, two request-wide authority indexes, four-wide catalog inspection, eight-wide global hanging-probe admission, consumer-scoped cache pruning, bounded inspection timeout, exact terminal_gone absence, three-poll fresh-presence reconciliation, cold handle-qualified contact loss, slow foreground completion, callback isolation, live/unverifiable/exited/hidden-tab/legacy projection, encrypted desktop/CLI capability advertisement, and cross-version terminal wire compatibility. Full mobile typecheck and 3,674 tests passed separately."
},
{
"date": "2026-08-13",
@@ -10704,7 +10715,7 @@
},
"performanceBudget": {
"required": true,
"evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, a hung provider retains at most one unresolved probe per identity, and every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path."
"evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, and at most eight unresolved provider probes can exist across PTYs even when their promises never settle. Restored projection builds one O(PTYs + persisted surfaces + restored rows) authority index before inspection and one after it, maps catalog work four-wide, and prunes entries owned only by disappeared restored hooks without pruning shared foreground evidence. Every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path."
},
"promotionCriteria": [
"Collect 100 consecutive passes or 14 days of stable CI history on macOS, Linux, and Windows.",
+161 -9
View File
@@ -1049,6 +1049,7 @@ const TEST_FOLDER_WORKSPACE_KEY = `folder:${TEST_FOLDER_WORKSPACE_ID}`
const TEST_FOLDER_WORKSPACE_PATH = '/tmp/platform'
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
const HEADLESS_LEAF_ID = '11111111-1111-4111-8111-111111111111'
const RESTORED_AGENT_INSPECTION_TEST_CONCURRENCY = 4
const HEADLESS_SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222'
const HEADLESS_THIRD_LEAF_ID = '33333333-3333-4333-8333-333333333333'
const RESTORED_AUTHORITY_TOKEN = 'restored-authority-secret'
@@ -1693,6 +1694,37 @@ function makeWorkspaceSessionWithHeadlessTerminal(
}
}
function makePersistedRestoredAgentSession(args: {
tabId: string
leafId: string
ptyId: string
incarnationId: string
}): WorkspaceSessionState {
const paneKey = makePaneKey(args.tabId, args.leafId)
return makeWorkspaceSessionWithHeadlessTerminal({
activeTabId: args.tabId,
activeTabIdByWorktree: { [TEST_WORKTREE_ID]: args.tabId },
tabsByWorktree: {
[TEST_WORKTREE_ID]: [
{
id: args.tabId,
ptyId: args.ptyId,
worktreeId: TEST_WORKTREE_ID,
title: 'Persisted Agent',
customTitle: null,
color: null,
sortOrder: 0,
createdAt: 1
}
]
},
terminalLayoutsByTabId: {
[args.tabId]: makeHeadlessTerminalLayout({ [args.leafId]: args.ptyId })
},
terminalPtyIncarnationsByPaneKey: { [paneKey]: args.incarnationId }
})
}
computeWorktreePathMock.mockImplementation(
(
sanitizedName: string,
@@ -37991,6 +38023,78 @@ describe('OrcaRuntimeService', () => {
expect(inspectProcess).not.toHaveBeenCalled()
})
it('indexes restored authority once per snapshot and bounds request inspection fanout', async () => {
const rowCount = 12
const tabs = Array.from({ length: rowCount }, (_, index) => ({
id: `tab-${index}`,
ptyId: `persisted-pty-${index}`,
worktreeId: TEST_WORKTREE_ID,
title: `Agent ${index}`,
customTitle: null,
color: null,
sortOrder: index,
createdAt: index + 1
}))
const terminalLayoutsByTabId = Object.fromEntries(
tabs.map((tab) => [tab.id, makeHeadlessTerminalLayout({ [HEADLESS_LEAF_ID]: tab.ptyId })])
)
const terminalPtyIncarnationsByPaneKey = Object.fromEntries(
tabs.map((tab, index) => [makePaneKey(tab.id, HEADLESS_LEAF_ID), `incarnation-${index}`])
)
const session = makeWorkspaceSessionWithHeadlessTerminal({
tabsByWorktree: { [TEST_WORKTREE_ID]: tabs },
terminalLayoutsByTabId,
terminalPtyIncarnationsByPaneKey
})
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
const getWorkspaceSession = vi.fn(runtimeStore.getWorkspaceSession)
let activeInspections = 0
let maxActiveInspections = 0
const inspectProcess = vi.fn(async () => {
activeInspections += 1
maxActiveInspections = Math.max(maxActiveInspections, activeInspections)
await Promise.resolve()
activeInspections -= 1
return { foregroundProcess: 'codex', hasChildProcesses: true }
})
const now = Date.now()
const runtime = new OrcaRuntimeService(
{ ...runtimeStore, getWorkspaceSession } as never,
undefined,
{
getAgentStatusSnapshot: () =>
tabs.map((tab, index) => ({
paneKey: makePaneKey(tab.id, HEADLESS_LEAF_ID),
worktreeId: TEST_WORKTREE_ID,
tabId: tab.id,
state: 'working' as const,
prompt: `restored agent ${index}`,
agentType: 'codex',
connectionId: null,
receivedAt: now + index,
stateStartedAt: now - rowCount + index,
restoredUnconfirmed: true
}))
}
)
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => 'codex'),
inspectProcess,
listProcesses: vi.fn(async () => [])
})
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
expect(
worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents
).toHaveLength(rowCount)
expect(inspectProcess).toHaveBeenCalledTimes(rowCount)
expect(maxActiveInspections).toBe(RESTORED_AGENT_INSPECTION_TEST_CONCURRENCY)
expect(getWorkspaceSession.mock.calls.length).toBeLessThanOrEqual(4)
})
it('omits a cold restored row only after terminal_gone confirms exact absence', async () => {
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
@@ -38085,6 +38189,14 @@ describe('OrcaRuntimeService', () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(
makePersistedRestoredAgentSession({
tabId: 'tab-1',
leafId,
ptyId: 'pty-1',
incarnationId
})
)
const now = Date.now()
const inspectProcess = vi.fn(async () => {
throw new Error('terminal_gone')
@@ -38102,7 +38214,7 @@ describe('OrcaRuntimeService', () => {
}
])
.mockRejectedValueOnce(new Error('socket_closed'))
const runtime = new OrcaRuntimeService(store, undefined, {
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
@@ -38183,12 +38295,21 @@ describe('OrcaRuntimeService', () => {
it('projects a restored agent after the execution host confirms its exact PTY is live', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(
makePersistedRestoredAgentSession({
tabId: 'tab-1',
leafId,
ptyId: 'pty-1',
incarnationId
})
)
const now = Date.now()
const inspectProcess = vi.fn(async () => ({
foregroundProcess: 'node /opt/codex/bin/codex',
hasChildProcesses: true
}))
const runtime = new OrcaRuntimeService(store, undefined, {
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
@@ -38209,7 +38330,9 @@ describe('OrcaRuntimeService', () => {
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => 'node /opt/codex/bin/codex'),
inspectProcess,
listProcesses: vi.fn(async () => [{ id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'Codex' }])
listProcesses: vi.fn(async () => [
{ id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'Codex', incarnationId }
])
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
@@ -38403,6 +38526,15 @@ describe('OrcaRuntimeService', () => {
it('fences a restored-agent inspection across PTY reincarnation', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const incarnationId = 'incarnation-a'
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(
makePersistedRestoredAgentSession({
tabId: 'tab-1',
leafId,
ptyId: 'pty-1',
incarnationId
})
)
const now = Date.now()
let resolveInspection!: (value: {
foregroundProcess: string
@@ -38414,7 +38546,7 @@ describe('OrcaRuntimeService', () => {
resolveInspection = resolve
})
)
const runtime = new OrcaRuntimeService(store, undefined, {
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
@@ -38440,7 +38572,7 @@ describe('OrcaRuntimeService', () => {
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
tabId: 'tab-1',
leafId,
incarnationId: 'incarnation-a'
incarnationId
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
@@ -38494,6 +38626,15 @@ describe('OrcaRuntimeService', () => {
it('does not publish a restored hook snapshot replaced during inspection', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const incarnationId = 'incarnation-a'
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(
makePersistedRestoredAgentSession({
tabId: 'tab-1',
leafId,
ptyId: 'pty-1',
incarnationId
})
)
const now = Date.now()
let rows: AgentStatusIpcPayload[] = [
{
@@ -38519,7 +38660,7 @@ describe('OrcaRuntimeService', () => {
resolveInspection = resolve
})
)
const runtime = new OrcaRuntimeService(store, undefined, {
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => rows
})
runtime.setPtyController({
@@ -38532,7 +38673,7 @@ describe('OrcaRuntimeService', () => {
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
tabId: 'tab-1',
leafId,
incarnationId: 'incarnation-a'
incarnationId
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
@@ -38570,12 +38711,21 @@ describe('OrcaRuntimeService', () => {
it('omits an exited restored agent after the exact PTY is confirmed shell-only', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(
makePersistedRestoredAgentSession({
tabId: 'tab-1',
leafId,
ptyId: 'pty-1',
incarnationId
})
)
const now = Date.now()
const inspectProcess = vi.fn(async () => ({
foregroundProcess: 'zsh',
hasChildProcesses: false
}))
const runtime = new OrcaRuntimeService(store, undefined, {
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
@@ -38596,7 +38746,9 @@ describe('OrcaRuntimeService', () => {
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => 'zsh'),
inspectProcess,
listProcesses: vi.fn(async () => [{ id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'zsh' }])
listProcesses: vi.fn(async () => [
{ id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'zsh', incarnationId }
])
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
+246 -104
View File
@@ -660,7 +660,11 @@ import type { RuntimeBrowserCommands } from './orca-runtime-browser'
import { createRuntimeBrowserCommands } from './runtime-browser-commands-factory'
import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency'
import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity'
import { PtyProcessLivenessBroker } from './pty-process-liveness-broker'
import {
PtyProcessLivenessBroker,
type PtyProcessLivenessEvidence
} from './pty-process-liveness-broker'
import { mapWithConcurrency } from '../../shared/map-with-concurrency'
import {
RestoredAgentAuthorityResolver,
sameRestoredAgentProcessAuthority,
@@ -1998,6 +2002,19 @@ type PtyControllerInventory = Readonly<{
queriedHostIds: ReadonlySet<ExecutionHostId>
}>
type PersistedRestoredAgentSurface = Readonly<{
ptyId: string
worktreeId: string
paneKey: string
incarnationId: string
}>
type RestoredAgentAuthoritySnapshot = Readonly<{
currentPtyByPaneKey: ReadonlyMap<string, RuntimePtyWorktreeRecord>
persistedSurfaceByKey: ReadonlyMap<string, PersistedRestoredAgentSurface | null>
inventory: PtyControllerInventory | null
}>
type WorktreeStartupDraftPaste = {
agent: TuiAgent
content: string
@@ -19090,9 +19107,10 @@ export class OrcaRuntimeService {
}
private getPtyProcessEvidenceIdentity(ptyId: string): string {
return (
this.ptysById.get(ptyId)?.incarnationId ??
`runtime:${this.runtimeId}:${this.getPtyLifecycleGeneration(ptyId)}`
return makePtyProcessEvidenceIdentity(
this.ptysById.get(ptyId)?.incarnationId ?? null,
this.runtimeId,
this.getPtyLifecycleGeneration(ptyId)
)
}
@@ -20591,14 +20609,76 @@ export class OrcaRuntimeService {
freshPtyInventory: PtyControllerInventory | null
): Promise<Map<string, RestoredAgentLivenessResolution>> {
const restoredEntries = entries.filter((entry) => entry.restoredUnconfirmed === true)
this.restoredAgentAuthority.retain(
new Set(restoredEntries.map((entry) => restoredAgentHookIdentity(entry)))
const retainedHookIdentities = new Set(
restoredEntries.map((entry) => restoredAgentHookIdentity(entry))
)
const resolved = await Promise.all(
restoredEntries.map(async (entry): Promise<[string, RestoredAgentLivenessResolution]> => {
const hookIdentity = restoredAgentHookIdentity(entry)
const authority = this.getRestoredAgentAuthority(entry, freshPtyInventory)
this.restoredAgentAuthority.retain(retainedHookIdentities)
const hooks = restoredEntries.map((entry) => ({
entry,
hookIdentity: restoredAgentHookIdentity(entry),
hook: this.getRestoredAgentAuthorityHook(entry)
}))
const initialSnapshot = this.buildRestoredAgentAuthoritySnapshot(
restoredEntries,
freshPtyInventory
)
const initial = hooks.map((item) => ({
...item,
authority: item.hook
? this.getRestoredAgentAuthority(item.hook, initialSnapshot)
: { binding: null, hasExactBinding: false }
}))
this.ptyProcessLiveness.retainConsumerEvidence(
RESTORED_AGENT_PROCESS_EVIDENCE_CONSUMER,
initial.flatMap(({ authority }) => {
const binding = authority.binding
return binding
? [{ ptyId: binding.ptyId, identity: this.getRestoredAgentEvidenceIdentity(binding) }]
: []
})
)
const controller = this.ptyController
const observed = await mapWithConcurrency(
initial,
RESTORED_AGENT_INSPECTION_CONCURRENCY,
async (
item
): Promise<{
item: (typeof initial)[number]
evidence: PtyProcessLivenessEvidence | null
settledLiveness: RestoredAgentLiveness | null
}> => {
const binding = item.authority.binding
if (!binding) {
return { item, evidence: null, settledLiveness: 'unverifiable' }
}
if (this.isPtyKnownExited(binding.ptyId)) {
return { item, evidence: null, settledLiveness: 'exited' }
}
if (!controller) {
return { item, evidence: null, settledLiveness: 'unverifiable' }
}
const evidence = await this.ptyProcessLiveness.inspect({
source: controller,
ptyId: binding.ptyId,
identity: this.getRestoredAgentEvidenceIdentity(binding),
deadline,
owningInventoryObservedPty: this.inventoryProvesRestoredBindingPresent(
freshPtyInventory,
binding
),
consumerId: RESTORED_AGENT_PROCESS_EVIDENCE_CONSUMER
})
return { item, evidence, settledLiveness: null }
}
)
const validationSnapshot = this.buildRestoredAgentAuthoritySnapshot(
restoredEntries,
freshPtyInventory
)
return new Map(
observed.map(({ item, evidence, settledLiveness }) => {
const { entry, hookIdentity, authority, hook } = item
const result = (
liveness: RestoredAgentLiveness,
hasExactBinding = authority.hasExactBinding
@@ -20606,37 +20686,23 @@ export class OrcaRuntimeService {
entry.paneKey,
{ liveness, hookIdentity, hasExactBinding }
]
if (!binding) {
return result('unverifiable')
if (settledLiveness) {
return result(settledLiveness)
}
if (this.isPtyKnownExited(binding.ptyId)) {
return result('exited')
}
const controller = this.ptyController
if (!controller) {
return result('unverifiable')
}
const evidence = await this.ptyProcessLiveness.inspect({
source: controller,
ptyId: binding.ptyId,
identity:
binding.incarnationId ?? `runtime:${this.runtimeId}:${binding.lifecycleGeneration}`,
deadline,
owningInventoryObservedPty: this.inventoryProvesRestoredBindingPresent(
freshPtyInventory,
binding
)
})
const currentAuthority = this.getRestoredAgentAuthority(entry, freshPtyInventory).binding
const binding = authority.binding
const currentAuthority = hook
? this.getRestoredAgentAuthority(hook, validationSnapshot).binding
: null
if (
controller !== this.ptyController ||
!binding ||
!currentAuthority ||
!sameRestoredAgentProcessAuthority(currentAuthority, binding)
) {
return result('unverifiable')
}
if (evidence.status !== 'live') {
return result(evidence.status)
if (!evidence || evidence.status !== 'live') {
return result(evidence?.status ?? 'unverifiable')
}
if (binding.source === 'persisted') {
return result('unverifiable')
@@ -20654,85 +20720,72 @@ export class OrcaRuntimeService {
)
})
)
return new Map(resolved)
}
private getRestoredAgentAuthority(
entry: AgentStatusIpcPayload,
inventory: PtyControllerInventory | null
hook: RestoredAgentAuthorityHook,
snapshot: RestoredAgentAuthoritySnapshot
) {
const hook = this.getRestoredAgentAuthorityHook(entry)
if (!hook) {
return { binding: null, hasExactBinding: false }
}
return this.restoredAgentAuthority.resolve({
hook,
current: this.getCurrentRestoredAgentBinding(entry, hook, inventory),
persisted: this.getPersistedRestoredAgentPtyBinding(entry, hook, inventory)
current: this.getCurrentRestoredAgentBinding(hook, snapshot),
persisted: this.getPersistedRestoredAgentPtyBinding(hook, snapshot)
})
}
private getPersistedRestoredAgentPtyBinding(
entry: AgentStatusIpcPayload,
hook: RestoredAgentAuthorityHook,
inventory: PtyControllerInventory | null
snapshot: RestoredAgentAuthoritySnapshot
): RestoredAgentAuthorityBinding | null {
if (!entry.worktreeId) {
const binding = snapshot.persistedSurfaceByKey.get(
restoredAgentSurfaceKey(hook.hostKey, hook.worktreeKey, hook.paneKey)
)
if (!binding) {
return null
}
const session = this.getWorkspaceSessionForWorktree(entry.worktreeId)
for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) {
if (
binding.paneKey === entry.paneKey &&
runtimeWorktreeIdsEqual(binding.worktreeId, entry.worktreeId)
) {
const controllerIdentity = inventory?.terminalIdentityByPtyId.get(ptyId)
const encodedHost = getPtyExecutionHost(ptyId)
if (
encodedHost === 'foreign' ||
(encodedHost !== null && encodedHost !== hook.hostKey) ||
(hook.hostKey.startsWith('wsl:') &&
controllerIdentity?.wslDistro?.toLowerCase() !== hook.hostKey.slice(4))
) {
return null
}
return {
ptyId,
incarnationId: binding.incarnationId,
lifecycleGeneration: this.getPtyLifecycleGeneration(ptyId),
source: 'persisted',
paneKey: binding.paneKey,
worktreeKey: hook.worktreeKey,
hostKey: hook.hostKey,
...((controllerIdentity?.handle ?? hook.terminalHandle)
? { terminalHandle: controllerIdentity?.handle ?? hook.terminalHandle }
: {})
}
}
const controllerIdentity = snapshot.inventory?.terminalIdentityByPtyId.get(binding.ptyId)
const encodedHost = getPtyExecutionHost(binding.ptyId)
if (
encodedHost === 'foreign' ||
(encodedHost !== null && encodedHost !== hook.hostKey) ||
(hook.hostKey.startsWith('wsl:') &&
controllerIdentity?.wslDistro?.toLowerCase() !== hook.hostKey.slice(4))
) {
return null
}
return {
ptyId: binding.ptyId,
incarnationId: binding.incarnationId,
lifecycleGeneration: this.getPtyLifecycleGeneration(binding.ptyId),
source: 'persisted',
paneKey: binding.paneKey,
worktreeKey: hook.worktreeKey,
hostKey: hook.hostKey,
...((controllerIdentity?.handle ?? hook.terminalHandle)
? { terminalHandle: controllerIdentity?.handle ?? hook.terminalHandle }
: {})
}
return null
}
private getCurrentRestoredAgentBinding(
entry: AgentStatusIpcPayload,
hook: RestoredAgentAuthorityHook,
inventory: PtyControllerInventory | null
snapshot: RestoredAgentAuthoritySnapshot
): RestoredAgentAuthorityBinding | null {
const current = this.getExactPtyRecordForPaneKey(entry.paneKey)
const current = snapshot.currentPtyByPaneKey.get(hook.paneKey)
if (
!current ||
!runtimeWorktreeIdsEqual(current.worktreeId, entry.worktreeId ?? '') ||
runtimeWorktreeIdentityKey(current.worktreeId) !== hook.worktreeKey ||
!this.ptyMatchesRestoredAgentHost(current, hook.hostKey)
) {
return null
}
const controllerIdentity = inventory?.terminalIdentityByPtyId.get(current.ptyId)
const controllerIdentity = snapshot.inventory?.terminalIdentityByPtyId.get(current.ptyId)
return {
ptyId: current.ptyId,
incarnationId: current.incarnationId,
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
source: 'current',
paneKey: entry.paneKey,
paneKey: hook.paneKey,
worktreeKey: hook.worktreeKey,
hostKey: hook.hostKey,
...((controllerIdentity?.handle ??
@@ -20748,6 +20801,100 @@ export class OrcaRuntimeService {
}
}
private buildRestoredAgentAuthoritySnapshot(
entries: readonly AgentStatusIpcPayload[],
inventory: PtyControllerInventory | null
): RestoredAgentAuthoritySnapshot {
const requestedPaneKeys = new Set(entries.map((entry) => entry.paneKey))
const currentCandidates = new Map<string, Set<RuntimePtyWorktreeRecord>>()
const addCurrent = (paneKey: string, pty: RuntimePtyWorktreeRecord | undefined): void => {
if (!pty || !requestedPaneKeys.has(paneKey)) {
return
}
const candidates = currentCandidates.get(paneKey) ?? new Set<RuntimePtyWorktreeRecord>()
candidates.add(pty)
currentCandidates.set(paneKey, candidates)
}
for (const paneKey of requestedPaneKeys) {
const parsed = parsePaneKey(paneKey)
const leaf = parsed
? this.leaves.get(this.getLeafKey(parsed.tabId, parsed.leafId))
: undefined
addCurrent(paneKey, leaf?.ptyId ? this.ptysById.get(leaf.ptyId) : undefined)
}
for (const pty of this.ptysById.values()) {
if (pty.paneKey) {
addCurrent(pty.paneKey, pty)
}
}
const currentPtyByPaneKey = new Map<string, RuntimePtyWorktreeRecord>()
for (const [paneKey, candidates] of currentCandidates) {
const connected = [...candidates].filter((pty) => pty.connected)
if (connected.length === 1) {
currentPtyByPaneKey.set(paneKey, connected[0]!)
} else if (connected.length === 0 && candidates.size === 1) {
currentPtyByPaneKey.set(paneKey, [...candidates][0]!)
}
}
const persistedSurfaceByKey = new Map<string, PersistedRestoredAgentSurface | null>()
const indexedSessionHostIds = new Set<ExecutionHostId>()
for (const entry of entries) {
if (!entry.worktreeId) {
continue
}
const sessionHostId = this.tryGetWorkspaceSessionHostIdForWorktree(entry.worktreeId)
if (!sessionHostId || indexedSessionHostIds.has(sessionHostId)) {
continue
}
indexedSessionHostIds.add(sessionHostId)
const session = this.store?.getWorkspaceSession?.(sessionHostId)
for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) {
const hostKeys = this.getPersistedRestoredAgentHostKeys(sessionHostId, ptyId, inventory)
for (const hostKey of hostKeys) {
const key = restoredAgentSurfaceKey(
hostKey,
runtimeWorktreeIdentityKey(binding.worktreeId),
binding.paneKey
)
const candidate = { ptyId, ...binding }
const existing = persistedSurfaceByKey.get(key)
if (existing === undefined) {
persistedSurfaceByKey.set(key, candidate)
} else if (
existing !== null &&
(existing.ptyId !== candidate.ptyId ||
existing.incarnationId !== candidate.incarnationId)
) {
persistedSurfaceByKey.set(key, null)
}
}
}
}
return { currentPtyByPaneKey, persistedSurfaceByKey, inventory }
}
private getPersistedRestoredAgentHostKeys(
sessionHostId: ExecutionHostId,
ptyId: string,
inventory: PtyControllerInventory | null
): readonly string[] {
const encodedHost = getPtyExecutionHost(ptyId)
if (encodedHost && encodedHost !== 'foreign') {
return [encodedHost]
}
const wslDistro = inventory?.terminalIdentityByPtyId.get(ptyId)?.wslDistro?.trim()
return wslDistro ? [sessionHostId, `wsl:${wslDistro.toLowerCase()}`] : [sessionHostId]
}
private getRestoredAgentEvidenceIdentity(binding: RestoredAgentAuthorityBinding): string {
return makePtyProcessEvidenceIdentity(
binding.incarnationId,
this.runtimeId,
binding.lifecycleGeneration
)
}
private getRestoredAgentAuthorityHook(
entry: AgentStatusIpcPayload
): RestoredAgentAuthorityHook | null {
@@ -20781,29 +20928,6 @@ export class OrcaRuntimeService {
}
}
private getExactPtyRecordForPaneKey(paneKey: string): RuntimePtyWorktreeRecord | null {
const parsed = parsePaneKey(paneKey)
const leaf = parsed ? this.leaves.get(this.getLeafKey(parsed.tabId, parsed.leafId)) : undefined
const candidates = new Set<RuntimePtyWorktreeRecord>()
if (leaf?.ptyId) {
const pty = this.ptysById.get(leaf.ptyId)
if (pty) {
candidates.add(pty)
}
}
for (const pty of this.ptysById.values()) {
if (pty.paneKey === paneKey) {
candidates.add(pty)
}
}
const connected = [...candidates].filter((pty) => pty.connected)
return connected.length === 1
? connected[0]!
: candidates.size === 1
? [...candidates][0]!
: null
}
private ptyMatchesRestoredAgentHost(pty: RuntimePtyWorktreeRecord, hostKey: string): boolean {
const encodedHost = getPtyExecutionHost(pty.ptyId)
if (encodedHost === 'foreign') {
@@ -39025,6 +39149,8 @@ export function resolveWorktreeScanCacheTtlMs(repo: Pick<Repo, 'path' | 'connect
}
const PTY_CONTROLLER_LIST_TIMEOUT_MS = 3000
const PTY_PROCESS_INSPECTION_TIMEOUT_MS = 1_000
const RESTORED_AGENT_INSPECTION_CONCURRENCY = 4
const RESTORED_AGENT_PROCESS_EVIDENCE_CONSUMER = 'restored-agent'
// Why: the slice of the list budget reserved for the aggregate to collect the providers
// that answered after a stalled one gives up.
const PTY_CONTROLLER_LIST_PROVIDER_MARGIN_MS = 500
@@ -40867,6 +40993,22 @@ function restoredAgentHookIdentity(entry: AgentStatusIpcPayload): string {
])
}
function restoredAgentSurfaceKey(hostKey: string, worktreeKey: string, paneKey: string): string {
return JSON.stringify([hostKey, worktreeKey, paneKey])
}
function makePtyProcessEvidenceIdentity(
incarnationId: string | null,
runtimeId: string,
lifecycleGeneration: number
): string {
return JSON.stringify(
incarnationId === null
? ['runtime-generation', runtimeId, lifecycleGeneration]
: ['incarnation', incarnationId]
)
}
function indexPersistedPtyWorktreeBindings(
session: WorkspaceSessionState | null | undefined
): ReadonlyMap<string, string> {
@@ -0,0 +1,60 @@
export type PtyProcessInspectionSource = {
getForegroundProcess(ptyId: string): Promise<string | null>
inspectProcess?: (ptyId: string) => Promise<{
foregroundProcess: string | null
hasChildProcesses: boolean
unavailable?: true
}>
hasChildProcesses?: (ptyId: string) => Promise<boolean>
}
export type PtyProcessLivenessEvidence =
| {
status: 'live'
foregroundProcess: string | null
hasChildProcesses: boolean | null
}
| { status: 'unverifiable'; reason: string }
| { status: 'exited' }
export async function inspectPtyProcess(
source: PtyProcessInspectionSource,
ptyId: string
): Promise<PtyProcessLivenessEvidence> {
try {
if (source.inspectProcess) {
const inspection = await source.inspectProcess(ptyId)
return inspection.unavailable
? { status: 'unverifiable', reason: 'process inspection unavailable' }
: {
status: 'live',
foregroundProcess: inspection.foregroundProcess,
hasChildProcesses: inspection.hasChildProcesses
}
}
const foregroundProcess = await source.getForegroundProcess(ptyId)
return {
status: 'live',
foregroundProcess,
hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null
}
} catch (error) {
if (isTerminalGoneError(error)) {
return { status: 'exited' }
}
throw error
}
}
export function describeProcessInspectionError(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}
function isTerminalGoneError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error)
const code =
error && typeof error === 'object' && 'code' in error
? String((error as { code?: unknown }).code)
: ''
return message === 'terminal_gone' || code === 'terminal_gone'
}
@@ -48,6 +48,80 @@ describe('PtyProcessLivenessBroker', () => {
expect(inspectProcess).toHaveBeenCalledOnce()
})
it('bounds hanging probes across PTYs and prunes retired consumer entries', async () => {
vi.useFakeTimers()
const resolvers: ((value: {
foregroundProcess: string
hasChildProcesses: boolean
}) => void)[] = []
const inspectProcess = vi.fn(
() =>
new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => {
resolvers.push(resolve)
})
)
const source = { getForegroundProcess: vi.fn(async () => null), inspectProcess }
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100, maxConcurrentProbes: 2 })
const reads = Array.from({ length: 5 }, (_, index) =>
broker.inspect({
source,
ptyId: `pty-${index}`,
identity: `incarnation-${index}`,
consumerId: 'restored-agent'
})
)
await vi.advanceTimersByTimeAsync(100)
await Promise.all(reads)
expect(inspectProcess).toHaveBeenCalledTimes(2)
expect(broker.getActiveProbeCount()).toBe(2)
expect(broker.getEntryCount()).toBe(2)
broker.retainConsumerEvidence('restored-agent', [])
expect(broker.getEntryCount()).toBe(0)
expect(broker.getActiveProbeCount()).toBe(2)
for (const resolve of resolvers) {
resolve({ foregroundProcess: 'codex', hasChildProcesses: true })
}
await vi.runAllTimersAsync()
expect(broker.getActiveProbeCount()).toBe(0)
})
it('does not prune evidence shared with an unscoped broker consumer', async () => {
let resolveInspection!: (value: {
foregroundProcess: string
hasChildProcesses: boolean
}) => void
const inspection = new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>(
(resolve) => {
resolveInspection = resolve
}
)
const source = {
getForegroundProcess: vi.fn(async () => null),
inspectProcess: vi.fn(() => inspection)
}
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 })
const restored = broker.inspect({
source,
ptyId: 'pty-1',
identity: 'incarnation-a',
consumerId: 'restored-agent'
})
const foreground = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
broker.retainConsumerEvidence('restored-agent', [])
expect(broker.getEntryCount()).toBe(1)
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
await expect(Promise.all([restored, foreground])).resolves.toEqual([
expect.objectContaining({ status: 'live' }),
expect.objectContaining({ status: 'live' })
])
})
it('caches live evidence across sequential clients', async () => {
const inspectProcess = vi.fn(async () => ({
foregroundProcess: 'codex',
+81 -59
View File
@@ -1,25 +1,20 @@
export type PtyProcessInspectionSource = {
getForegroundProcess(ptyId: string): Promise<string | null>
inspectProcess?: (ptyId: string) => Promise<{
foregroundProcess: string | null
hasChildProcesses: boolean
unavailable?: true
}>
hasChildProcesses?: (ptyId: string) => Promise<boolean>
}
import {
describeProcessInspectionError,
inspectPtyProcess,
type PtyProcessInspectionSource,
type PtyProcessLivenessEvidence
} from './pty-process-inspection'
export type PtyProcessLivenessEvidence =
| {
status: 'live'
foregroundProcess: string | null
hasChildProcesses: boolean | null
}
| { status: 'unverifiable'; reason: string }
| { status: 'exited' }
export type {
PtyProcessInspectionSource,
PtyProcessLivenessEvidence
} from './pty-process-inspection'
type PtyProcessEvidenceEntry = {
source: PtyProcessInspectionSource
identity: string
consumerIds: Set<string>
hasUnscopedConsumer: boolean
freshness: number
owningInventoryObservedPty: boolean
failureCount: number
@@ -31,6 +26,7 @@ type PtyProcessEvidenceEntry = {
export type PtyProcessLivenessBrokerOptions = {
timeoutMs: number
maxConcurrentProbes?: number
liveTtlMs?: number
unavailableBackoffBaseMs?: number
unavailableBackoffMaxMs?: number
@@ -41,10 +37,12 @@ export type PtyProcessLivenessBrokerOptions = {
const DEFAULT_LIVE_TTL_MS = 10_000
const DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS = 3_000
const DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS = 30_000
const DEFAULT_MAX_CONCURRENT_PROBES = 8
export class PtyProcessLivenessBroker {
private readonly entries = new Map<string, PtyProcessEvidenceEntry>()
private readonly now: () => number
private activeProbeCount = 0
constructor(private readonly options: PtyProcessLivenessBrokerOptions) {
this.now = options.now ?? Date.now
@@ -59,6 +57,7 @@ export class PtyProcessLivenessBroker {
deadline?: number
waitForSettlement?: boolean
owningInventoryObservedPty?: boolean
consumerId?: string
}): Promise<PtyProcessLivenessEvidence> {
const freshness = args.freshness ?? 0
const waitMs = args.waitForSettlement
@@ -71,6 +70,9 @@ export class PtyProcessLivenessBroker {
)
)
const existing = this.entries.get(args.ptyId)
if (existing?.source === args.source && existing.identity === args.identity) {
this.rememberConsumer(existing, args.consumerId)
}
if (
existing?.source === args.source &&
existing.identity === args.identity &&
@@ -121,6 +123,12 @@ export class PtyProcessLivenessBroker {
if (waitMs === 0) {
return Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
}
if (this.activeProbeCount >= this.maxConcurrentProbes()) {
return Promise.resolve({
status: 'unverifiable',
reason: 'process inspection capacity unavailable'
})
}
const failureCount =
existing?.source === args.source && existing.identity === args.identity
@@ -129,6 +137,8 @@ export class PtyProcessLivenessBroker {
const entry: PtyProcessEvidenceEntry = {
source: args.source,
identity: args.identity,
consumerIds: new Set(args.consumerId ? [args.consumerId] : []),
hasUnscopedConsumer: args.consumerId === undefined,
freshness,
owningInventoryObservedPty: args.owningInventoryObservedPty === true,
failureCount,
@@ -137,7 +147,8 @@ export class PtyProcessLivenessBroker {
timedOut: false,
probe: null
}
const probe = this.runProbe(args.source, args.ptyId)
this.activeProbeCount += 1
const probe = inspectPtyProcess(args.source, args.ptyId)
.then((evidence) => {
const reconciled =
evidence.status === 'exited' && entry.owningInventoryObservedPty
@@ -169,7 +180,10 @@ export class PtyProcessLivenessBroker {
entry.probe = null
entry.timedOut = false
entry.failureCount += 1
entry.evidence = { status: 'unverifiable', reason: describeError(error) }
entry.evidence = {
status: 'unverifiable',
reason: describeProcessInspectionError(error)
}
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
}
try {
@@ -177,9 +191,11 @@ export class PtyProcessLivenessBroker {
} catch {
// Diagnostic observers cannot change the authority verdict.
}
return { status: 'unverifiable', reason: describeError(error) }
return { status: 'unverifiable', reason: describeProcessInspectionError(error) }
})
entry.probe = probe
entry.probe = probe.finally(() => {
this.activeProbeCount -= 1
})
this.entries.set(args.ptyId, entry)
return this.waitForProbe(args.ptyId, entry, waitMs)
}
@@ -192,6 +208,27 @@ export class PtyProcessLivenessBroker {
this.entries.clear()
}
retainConsumerEvidence(
consumerId: string,
retained: readonly Readonly<{ ptyId: string; identity: string }>[]
): void {
const retainedKeys = new Set(
retained.map(({ ptyId, identity }) => processEvidenceKey(ptyId, identity))
)
for (const [ptyId, entry] of this.entries) {
if (
!entry.consumerIds.has(consumerId) ||
retainedKeys.has(processEvidenceKey(ptyId, entry.identity))
) {
continue
}
entry.consumerIds.delete(consumerId)
if (!entry.hasUnscopedConsumer && entry.consumerIds.size === 0) {
this.entries.delete(ptyId)
}
}
}
getPendingCount(): number {
let count = 0
for (const entry of this.entries.values()) {
@@ -202,33 +239,12 @@ export class PtyProcessLivenessBroker {
return count
}
private async runProbe(
source: PtyProcessInspectionSource,
ptyId: string
): Promise<PtyProcessLivenessEvidence> {
try {
if (source.inspectProcess) {
const inspection = await source.inspectProcess(ptyId)
return inspection.unavailable
? { status: 'unverifiable', reason: 'process inspection unavailable' }
: {
status: 'live',
foregroundProcess: inspection.foregroundProcess,
hasChildProcesses: inspection.hasChildProcesses
}
}
const foregroundProcess = await source.getForegroundProcess(ptyId)
return {
status: 'live',
foregroundProcess,
hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null
}
} catch (error) {
if (isTerminalGoneError(error)) {
return { status: 'exited' }
}
throw error
}
getActiveProbeCount(): number {
return this.activeProbeCount
}
getEntryCount(): number {
return this.entries.size
}
private waitForProbe(
@@ -265,6 +281,21 @@ export class PtyProcessLivenessBroker {
return Math.min(max, base * 2 ** Math.max(0, failureCount - 1))
}
private maxConcurrentProbes(): number {
const configured = this.options.maxConcurrentProbes ?? DEFAULT_MAX_CONCURRENT_PROBES
return Number.isFinite(configured) && configured > 0
? Math.max(1, Math.floor(configured))
: DEFAULT_MAX_CONCURRENT_PROBES
}
private rememberConsumer(entry: PtyProcessEvidenceEntry, consumerId: string | undefined): void {
if (consumerId === undefined) {
entry.hasUnscopedConsumer = true
} else {
entry.consumerIds.add(consumerId)
}
}
private storeUnverifiable(entry: PtyProcessEvidenceEntry, reason: string): void {
entry.failureCount += 1
entry.evidence = { status: 'unverifiable', reason }
@@ -272,15 +303,6 @@ export class PtyProcessLivenessBroker {
}
}
function isTerminalGoneError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error)
const code =
error && typeof error === 'object' && 'code' in error
? String((error as { code?: unknown }).code)
: ''
return message === 'terminal_gone' || code === 'terminal_gone'
}
function describeError(error: unknown): string {
return error instanceof Error ? error.message : String(error)
function processEvidenceKey(ptyId: string, identity: string): string {
return JSON.stringify([ptyId, identity])
}
@@ -42,9 +42,13 @@ describe('RestoredAgentAuthorityResolver', () => {
it('keeps one hook fenced to its first process incarnation', () => {
const resolver = new RestoredAgentAuthorityResolver()
expect(resolver.resolve({ hook, current: binding(), persisted: null }).binding).toEqual(
binding()
)
expect(
resolver.resolve({
hook,
current: binding(),
persisted: binding({ source: 'persisted' })
}).binding
).toEqual(binding())
expect(
resolver.resolve({
@@ -57,7 +61,11 @@ describe('RestoredAgentAuthorityResolver', () => {
it('treats lifecycle generations as fallback identity when incarnation exists', () => {
const resolver = new RestoredAgentAuthorityResolver()
resolver.resolve({ hook, current: binding(), persisted: null })
resolver.resolve({
hook,
current: binding(),
persisted: binding({ source: 'persisted' })
})
expect(
resolver.resolve({
@@ -68,6 +76,91 @@ describe('RestoredAgentAuthorityResolver', () => {
).toEqual(binding({ lifecycleGeneration: 2 }))
})
it('does not confuse a generation-shaped incarnation with fallback identity', () => {
const resolver = new RestoredAgentAuthorityResolver()
resolver.resolve({
hook,
current: binding({ incarnationId: null, lifecycleGeneration: 1 }),
persisted: binding({
incarnationId: null,
lifecycleGeneration: 1,
source: 'persisted'
})
})
expect(
resolver.resolve({
hook,
current: binding({ incarnationId: 'generation:1', lifecycleGeneration: 99 }),
persisted: null
})
).toEqual({ binding: null, hasExactBinding: false })
})
it('keeps process authority stable when a terminal handle is learned', () => {
const resolver = new RestoredAgentAuthorityResolver()
resolver.resolve({
hook,
current: binding(),
persisted: binding({ source: 'persisted' })
})
expect(
resolver.resolve({
hook,
current: binding({ terminalHandle: 'term_learned' }),
persisted: null
}).binding
).toEqual(binding({ terminalHandle: 'term_learned' }))
})
it('uses a hook terminal handle as a constraint when one is present', () => {
const resolver = new RestoredAgentAuthorityResolver()
const handleHook = { ...hook, terminalHandle: 'term_expected' }
expect(
resolver.resolve({
hook: handleHook,
current: binding({ terminalHandle: 'term_other' }),
persisted: null
})
).toEqual({ binding: null, hasExactBinding: false })
})
it('requires persisted authority before accepting a current binding', () => {
const resolver = new RestoredAgentAuthorityResolver()
expect(resolver.resolve({ hook, current: binding(), persisted: null })).toEqual({
binding: null,
hasExactBinding: false
})
})
it('seeds from persisted authority and later accepts matching current authority', () => {
const resolver = new RestoredAgentAuthorityResolver()
const persisted = binding({ source: 'persisted' })
expect(resolver.resolve({ hook, current: null, persisted })).toEqual({
binding: persisted,
hasExactBinding: true
})
expect(resolver.resolve({ hook, current: binding(), persisted: null }).binding).toEqual(
binding()
)
})
it('does not let a replacement current binding borrow first authority', () => {
const resolver = new RestoredAgentAuthorityResolver()
expect(
resolver.resolve({
hook,
current: binding({ ptyId: 'pty-replacement', incarnationId: 'incarnation-b' }),
persisted: binding({ source: 'persisted' })
})
).toEqual({ binding: null, hasExactBinding: false })
})
it('rejects disagreement between current and persisted bindings', () => {
const resolver = new RestoredAgentAuthorityResolver()
@@ -82,15 +175,20 @@ describe('RestoredAgentAuthorityResolver', () => {
it('forgets commitments after their restored hook disappears', () => {
const resolver = new RestoredAgentAuthorityResolver()
resolver.resolve({ hook, current: binding(), persisted: null })
resolver.resolve({
hook,
current: binding(),
persisted: binding({ source: 'persisted' })
})
resolver.retain(new Set())
const replacement = binding({ incarnationId: 'incarnation-b', source: 'persisted' })
expect(
resolver.resolve({
hook,
current: binding({ incarnationId: 'incarnation-b' }),
persisted: null
current: null,
persisted: replacement
}).binding
).toEqual(binding({ incarnationId: 'incarnation-b' }))
).toEqual(replacement)
})
})
@@ -35,12 +35,15 @@ export class RestoredAgentAuthorityResolver {
if (current && persisted && !sameRestoredAgentProcessAuthority(current, persisted)) {
return { binding: null, hasExactBinding: false }
}
const existing = this.commitmentByHookIdentity.get(args.hook.identity)
if (!existing && !persisted) {
return { binding: null, hasExactBinding: false }
}
const binding = current ?? persisted
if (!binding) {
return { binding: null, hasExactBinding: false }
}
const commitment = processAuthorityKey(binding)
const existing = this.commitmentByHookIdentity.get(args.hook.identity)
if (existing && existing !== commitment) {
return { binding: null, hasExactBinding: false }
}
@@ -83,7 +86,8 @@ function processAuthorityKey(binding: RestoredAgentAuthorityBinding): string {
binding.worktreeKey,
binding.paneKey,
binding.ptyId,
binding.incarnationId ?? `generation:${binding.lifecycleGeneration}`,
binding.terminalHandle ?? null
binding.incarnationId === null
? ['generation', binding.lifecycleGeneration]
: ['incarnation', binding.incarnationId]
])
}