From 77b364984eab21c60bcd9dadb89411d7584b24a6 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Sun, 23 Aug 2026 18:59:44 -0400 Subject: [PATCH] fix(runtime): fence restored agent liveness evidence --- config/reliability-gates.jsonc | 14 +- src/main/runtime/orca-runtime.test.ts | 306 +++++++++++++- src/main/runtime/orca-runtime.ts | 381 +++++++++++------- .../pty-process-liveness-broker.test.ts | 134 ++++++ .../runtime/pty-process-liveness-broker.ts | 250 ++++++++++++ 5 files changed, 922 insertions(+), 163 deletions(-) create mode 100644 src/main/runtime/pty-process-liveness-broker.test.ts create mode 100644 src/main/runtime/pty-process-liveness-broker.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 133a03b954b..b84b1b6025d 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10293,7 +10293,7 @@ "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts", - "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts", "cd mobile && pnpm exec vitest run src/worktree/agent-row-display.test.ts src/transport/stable-logical-rpc-client.test.ts", "pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1", "pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1", @@ -10305,6 +10305,7 @@ "src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", "src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts", "src/main/runtime/orca-runtime.test.ts", + "src/main/runtime/pty-process-liveness-broker.test.ts", "src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts", "src/cli/handlers/orchestration.test.ts", "src/cli/handlers/orchestration-lifecycle-rejection.test.ts", @@ -10356,7 +10357,10 @@ "retained renderer output remains readable through the recovered terminal", "capable worktree inventory projects restored live and unverifiable identities but omits exited shell-only agents", "a connected live PTY remains inventoried when its user-visible tab is missing", - "legacy clients retain the exact restored-row suppression without a process inspection" + "legacy clients retain the exact restored-row suppression", + "inspection timeout, transport loss, missing bindings, and stale incarnations remain unverifiable", + "terminal_gone is the only inspection rejection that retires the exact restored row", + "sequential inventory clients reuse bounded process evidence instead of polling each restored row" ] }, { @@ -10504,10 +10508,10 @@ "date": "2026-08-23", "runner": "local", "platform": "macos", - "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts", "result": "passed", "durationSeconds": 15.33, - "summary": "Seven files passed 1,264 tests with one skip, covering live, unverifiable, exited, hidden-tab, legacy and request-level capability projection plus every encrypted desktop/CLI client capability advertisement. Mobile typecheck and 25 focused mobile tests separately passed." + "summary": "Eight focused runtime and wire files passed 1,273 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, cold persisted binding, incarnation and hook replacement races, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, and encrypted desktop/CLI capability advertisement. Mobile typecheck and 25 focused mobile tests passed separately." }, { "date": "2026-08-13", @@ -10686,7 +10690,7 @@ }, "performanceBudget": { "required": true, - "evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. Capable worktree.ps requests inspect only restored-unconfirmed candidates, deduplicate concurrent and same-request reads by PTY, and do no inspection for legacy clients, confirmed rows, unrelated PTYs, or restored rows without a retained PTY. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path." + "evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, a hung provider retains at most one unresolved probe per identity, and every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path." }, "promotionCriteria": [ "Collect 100 consecutive passes or 14 days of stable CI history on macOS, Linux, and Windows.", diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 3deb1b5d837..8f968c7d7c2 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -22,7 +22,10 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state import type { WorkspaceLineage, WorktreeLineage } from '../../shared/worktree/lineage-types' import type { WorktreeMeta } from '../../shared/worktree/meta-types' import type { Worktree } from '../../shared/worktree/types' -import { AGENT_STATUS_STALE_AFTER_MS } from '../../shared/agent-status-types' +import { + AGENT_STATUS_STALE_AFTER_MS, + type AgentStatusIpcPayload +} from '../../shared/agent-status-types' import { reviewHeadRemoteRefComponent, REVIEW_HEAD_FETCH_TIMEOUT_MS @@ -9873,12 +9876,12 @@ describe('OrcaRuntimeService', () => { expect( ( runtime as unknown as { - ptyForegroundProcessReads: Map + ptyProcessLiveness: { getPendingCount(): number } } - ).ptyForegroundProcessReads.size + ).ptyProcessLiveness.getPendingCount() ).toBe(0) ) - await Promise.resolve() + await new Promise((resolve) => setImmediate(resolve)) getForegroundProcess.mockResolvedValueOnce('zsh') runtime.onPtyData('pty-1', '\x1b]0;other cwd\x07', 102) @@ -37744,12 +37747,146 @@ describe('OrcaRuntimeService', () => { leaves: [] }) - const { worktrees } = await runtime.getWorktreePs() + const { worktrees } = await runtime.getWorktreePs(undefined, true, false) const summary = worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID) expect(summary).toMatchObject({ hasHostSidebarActivity: false, status: 'inactive', agents: [] }) }) + it('keeps a restored row unverifiable when current PTY bookkeeping is missing', async () => { + const leafId = '33333333-3333-4333-8333-333333333333' + const paneKey = `tab-1:${leafId}` + const now = Date.now() + const runtime = new OrcaRuntimeService(store, undefined, { + getAgentStatusSnapshot: () => [ + { + paneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'tab-1', + state: 'working', + prompt: 'await provider adoption', + agentType: 'codex', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 100, + restoredUnconfirmed: true + } + ] + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-1', + worktreeId: TEST_WORKTREE_ID, + title: 'Codex', + activeLeafId: leafId, + layout: null + } + ], + leaves: [] + }) + + const { worktrees } = await runtime.getWorktreePs(undefined, true, true) + + expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([ + expect.objectContaining({ + paneKey, + restoredUnconfirmed: true, + agentLiveness: 'unverifiable' + }) + ]) + }) + + it('uses a cold persisted binding without certifying a replacement incarnation live', async () => { + const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID) + const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const session = makeWorkspaceSessionWithHeadlessTerminal({ + terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId } + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const now = Date.now() + const inspectProcess = vi.fn(async () => ({ + foregroundProcess: 'codex', + hasChildProcesses: true + })) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getAgentStatusSnapshot: () => [ + { + paneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'host-tab', + state: 'working', + prompt: 'cold persisted agent', + agentType: 'codex', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 100, + restoredUnconfirmed: true + } + ] + }) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: vi.fn(async () => 'codex'), + inspectProcess, + listProcesses: vi.fn(async () => []) + }) + + const { worktrees } = await runtime.getWorktreePs(undefined, true, true) + + expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([ + expect.objectContaining({ + paneKey, + restoredUnconfirmed: true, + agentLiveness: 'unverifiable' + }) + ]) + expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('persisted-pty') + }) + + it('omits a cold restored row only after terminal_gone confirms exact absence', async () => { + const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID) + const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const session = makeWorkspaceSessionWithHeadlessTerminal({ + terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId } + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const now = Date.now() + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + getAgentStatusSnapshot: () => [ + { + paneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'host-tab', + state: 'working', + prompt: 'completed while runtime was down', + agentType: 'codex', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 100, + restoredUnconfirmed: true + } + ] + }) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: vi.fn(async () => null), + inspectProcess: vi.fn(async () => { + throw new Error('terminal_gone') + }), + listProcesses: vi.fn(async () => []) + }) + + const { worktrees } = await runtime.getWorktreePs(undefined, true, true) + + expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual( + [] + ) + }) + it('projects a restored agent after the execution host confirms its exact PTY is live', async () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` @@ -37833,7 +37970,7 @@ describe('OrcaRuntimeService', () => { agentLiveness: 'live' }) ]) - expect(inspectProcess).toHaveBeenCalledTimes(2) + expect(inspectProcess).toHaveBeenCalledOnce() }) it('keeps a restored agent visible as unverifiable when PTY inspection loses contact', async () => { @@ -37900,6 +38037,161 @@ describe('OrcaRuntimeService', () => { ]) }) + it('fences a restored-agent inspection across PTY reincarnation', async () => { + const leafId = '33333333-3333-4333-8333-333333333333' + const paneKey = `tab-1:${leafId}` + const now = Date.now() + let resolveInspection!: (value: { + foregroundProcess: string + hasChildProcesses: boolean + }) => void + const inspectProcess = vi.fn( + () => + new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => { + resolveInspection = resolve + }) + ) + const runtime = new OrcaRuntimeService(store, undefined, { + getAgentStatusSnapshot: () => [ + { + paneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'tab-1', + state: 'working', + prompt: 'old incarnation', + agentType: 'codex', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 100, + restoredUnconfirmed: true + } + ] + }) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: vi.fn(async () => 'codex'), + inspectProcess, + listProcesses: vi.fn(async () => []) + }) + runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, { + tabId: 'tab-1', + leafId, + incarnationId: 'incarnation-a' + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-1', + worktreeId: TEST_WORKTREE_ID, + title: 'Codex', + activeLeafId: leafId, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-1', + worktreeId: TEST_WORKTREE_ID, + leafId, + paneRuntimeId: 1, + ptyId: 'pty-1' + } + ] + }) + + const inventory = runtime.getWorktreePs(undefined, true, true) + await vi.waitFor(() => expect(inspectProcess).toHaveBeenCalledOnce()) + runtime.onPtySpawned('pty-1', 'incarnation-b', { awaitsRegistration: false }) + resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true }) + + const { worktrees } = await inventory + expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([ + expect.objectContaining({ + paneKey, + restoredUnconfirmed: true, + agentLiveness: 'unverifiable' + }) + ]) + }) + + it('does not publish a restored hook snapshot replaced during inspection', async () => { + const leafId = '33333333-3333-4333-8333-333333333333' + const paneKey = `tab-1:${leafId}` + const now = Date.now() + let rows: AgentStatusIpcPayload[] = [ + { + paneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'tab-1', + state: 'working', + prompt: 'stale hook', + agentType: 'codex', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 100, + restoredUnconfirmed: true + } + ] + let resolveInspection!: (value: { + foregroundProcess: string + hasChildProcesses: boolean + }) => void + const inspectProcess = vi.fn( + () => + new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => { + resolveInspection = resolve + }) + ) + const runtime = new OrcaRuntimeService(store, undefined, { + getAgentStatusSnapshot: () => rows + }) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: vi.fn(async () => 'codex'), + inspectProcess, + listProcesses: vi.fn(async () => []) + }) + runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, { + tabId: 'tab-1', + leafId, + incarnationId: 'incarnation-a' + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-1', + worktreeId: TEST_WORKTREE_ID, + title: 'Codex', + activeLeafId: leafId, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-1', + worktreeId: TEST_WORKTREE_ID, + leafId, + paneRuntimeId: 1, + ptyId: 'pty-1' + } + ] + }) + + const inventory = runtime.getWorktreePs(undefined, true, true) + await vi.waitFor(() => expect(inspectProcess).toHaveBeenCalledOnce()) + rows = [] + resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true }) + + const { worktrees } = await inventory + expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual( + [] + ) + }) + it('omits an exited restored agent after the exact PTY is confirmed shell-only', async () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` @@ -38018,7 +38310,7 @@ describe('OrcaRuntimeService', () => { expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual( [] ) - expect(inspectProcess).not.toHaveBeenCalled() + expect(inspectProcess).toHaveBeenCalledOnce() }) it('uses mirrored tab ownership after a workspace rename instead of stale hook attribution', async () => { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 040e288cbb1..1fc27c9713b 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -660,6 +660,7 @@ import type { RuntimeBrowserCommands } from './orca-runtime-browser' import { createRuntimeBrowserCommands } from './runtime-browser-commands-factory' import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency' import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity' +import { PtyProcessLivenessBroker } from './pty-process-liveness-broker' import { buildHeadlessTerminalSplitLayout, countTerminalLayoutLeaves, @@ -1631,26 +1632,21 @@ type PtyForegroundProcessRead = { available: boolean } -type PtyForegroundProcessReadEntry = { - controller: RuntimePtyController - startedAfterTitleObservation: number - promise: Promise -} - -type RestoredAgentPtyInspection = { - controller: RuntimePtyController - foregroundProcess: string | null - hasChildProcesses: boolean - unavailable: boolean -} - -type RestoredAgentPtyInspectionEntry = { - controller: RuntimePtyController - promise: Promise -} - type RestoredAgentLiveness = 'live' | 'unverifiable' | 'exited' +type RestoredAgentLivenessResolution = { + liveness: RestoredAgentLiveness + hookIdentity: string + hasExactBinding: boolean +} + +type RestoredAgentPtyBinding = { + ptyId: string + incarnationId: PtyIncarnationId | null + lifecycleGeneration: number + source: 'current' | 'persisted' +} + function copySleepingAgentLaunchConfig( config: SleepingAgentLaunchConfig ): SleepingAgentLaunchConfig { @@ -3184,8 +3180,12 @@ export class OrcaRuntimeService { private worktreeAdminFingerprintProbes = new Set() private cloneInFlightByPath = new Map>() private ptyForegroundAgentRefreshes = new Map() - private ptyForegroundProcessReads = new Map() - private restoredAgentPtyInspections = new Map() + private readonly ptyProcessLiveness = new PtyProcessLivenessBroker({ + timeoutMs: PTY_PROCESS_INSPECTION_TIMEOUT_MS, + onInspectionError: (ptyId, error) => { + console.warn(`[runtime] PTY process inspection failed for ${ptyId}`, error) + } + }) private ptyDelayedForegroundSnapshotTitleObservations = new Map() // Why a set and not a timer: the intent is retired by the exit it explains, or // by the next lifecycle generation on that id (advancePtyLifecycleGeneration), @@ -6181,7 +6181,10 @@ export class OrcaRuntimeService { // Why: CLI terminal writes must go through the main-owned PTY registry // instead of tunneling back through renderer IPC, or live handles could // drift from the process they are supposed to control during reloads. - this.ptyController = controller + if (this.ptyController !== controller) { + this.ptyProcessLiveness.invalidateAll() + this.ptyController = controller + } } setNotifier(notifier: RuntimeNotifier | null): void { @@ -10795,6 +10798,12 @@ export class OrcaRuntimeService { incarnationId?: PtyIncarnationId, options: { awaitsRegistration?: boolean } = {} ): void { + const previousIncarnation = this.ptysById.get(ptyId)?.incarnationId + if (previousIncarnation && incarnationId && previousIncarnation !== incarnationId) { + this.advancePtyLifecycleGeneration(ptyId) + } else { + this.ptyProcessLiveness.invalidate(ptyId) + } this.forgetPtyLivenessVerdict(ptyId) if (options.awaitsRegistration !== false) { // Why: surface absence cannot distinguish an in-flight admission from a completed headless lifecycle. @@ -12262,6 +12271,7 @@ export class OrcaRuntimeService { private advancePtyLifecycleGeneration(ptyId: string): void { this.ptyLifecycleGenerationById.set(ptyId, this.nextPtyLifecycleGeneration++) + this.ptyProcessLiveness.invalidate(ptyId) // Why: a stop whose exit never arrived would otherwise stay armed across a // same-id respawn and label the NEXT process's crash an operator close — // the exact lie this cause model exists to remove. @@ -19050,73 +19060,44 @@ export class OrcaRuntimeService { private readPtyForegroundProcessFromController( ptyId: string, - afterTitleObservation = 0 + afterTitleObservation = 0, + options: { reuseSettled?: boolean } = {} ): Promise | null { const controller = this.ptyController if (!controller) { return null } - const pending = this.ptyForegroundProcessReads.get(ptyId) - if ( - pending?.controller === controller && - pending.startedAfterTitleObservation >= afterTitleObservation - ) { - return pending.promise - } - if (pending?.controller === controller) { - return pending.promise.then( - () => - this.readPtyForegroundProcessFromController(ptyId, afterTitleObservation) ?? { - controller, - process: null, - available: false - } - ) - } - const unavailable: PtyForegroundProcessRead = { - controller, - process: null, - available: false - } - let processRead: Promise - try { - processRead = Promise.resolve(controller.getForegroundProcess(ptyId)) - } catch { - const entry: PtyForegroundProcessReadEntry = { + const identity = this.getPtyProcessEvidenceIdentity(ptyId) + return this.ptyProcessLiveness + .inspect({ + source: controller, + ptyId, + identity, + freshness: afterTitleObservation, + reuseSettled: options.reuseSettled + }) + .then((evidence) => ({ controller, - startedAfterTitleObservation: afterTitleObservation, - promise: Promise.resolve(unavailable) - } - entry.promise = entry.promise.finally(() => { - if (this.ptyForegroundProcessReads.get(ptyId) === entry) { - this.ptyForegroundProcessReads.delete(ptyId) - } - }) - this.ptyForegroundProcessReads.set(ptyId, entry) - return entry.promise - } - let entry: PtyForegroundProcessReadEntry - const promise = processRead - .then((process) => ({ controller, process, available: true })) - .catch(() => unavailable) - .finally(() => { - if (this.ptyForegroundProcessReads.get(ptyId) === entry) { - this.ptyForegroundProcessReads.delete(ptyId) - } - }) - entry = { - controller, - startedAfterTitleObservation: afterTitleObservation, - promise - } - this.ptyForegroundProcessReads.set(ptyId, entry) - return entry.promise + process: evidence.status === 'live' ? evidence.foregroundProcess : null, + available: evidence.status === 'live' + })) + } + + private getPtyProcessEvidenceIdentity(ptyId: string): string { + return ( + this.ptysById.get(ptyId)?.incarnationId ?? + `runtime:${this.runtimeId}:${this.getPtyLifecycleGeneration(ptyId)}` + ) } private confirmPtyAgentExit(ptyId: string): void { const pty = this.ptysById.get(ptyId) const titleObservedAt = pty?.lastOscTitleAt ?? null - const foregroundRead = this.readPtyForegroundProcessFromController(ptyId, titleObservedAt ?? 0) + const foregroundRead = this.readPtyForegroundProcessFromController( + ptyId, + titleObservedAt ?? 0, + { reuseSettled: false } + ) if (!pty?.connected || !foregroundRead) { this.recordTerminalSideEffectFact(ptyId, { kind: 'agent-exited' }) return @@ -19996,6 +19977,7 @@ export class OrcaRuntimeService { visibilitySettings ) ) + const ptyProcessEvidenceDeadline = Date.now() + PTY_CONTROLLER_LIST_TIMEOUT_MS // Why: worktree.ps backs the mobile sidebar, so it must use the same // host-owned imported-worktree visibility gate as worktree.list/desktop. const freshPtyLiveness = await this.refreshPtyWorktreeRecordsFromController(resolvedWorktrees) @@ -20355,7 +20337,8 @@ export class OrcaRuntimeService { missingRuntimeWorktreeIds, mirroredWorktreeIdByTabId, connectedPtyEvidence, - restoredAgentPresenceSupported + restoredAgentPresenceSupported, + ptyProcessEvidenceDeadline ) const sorted = [...summaries.values()].sort(compareWorktreePs) @@ -20380,7 +20363,8 @@ export class OrcaRuntimeService { paneKeys: ReadonlySet ptyIds: ReadonlySet }, - restoredAgentPresenceSupported: boolean + restoredAgentPresenceSupported: boolean, + ptyProcessEvidenceDeadline: number ): Promise { // Why: most agents report via hooks (agent-hooks/server), not OSC, so the // hook snapshot is the primary source — same one the desktop sidebar reads. @@ -20404,6 +20388,7 @@ export class OrcaRuntimeService { updatedAt: number restoredUnconfirmed?: boolean agentLiveness?: Exclude + restoredExactBinding?: boolean } >() for (const snapshot of this.latestAgentStatusByPaneKey.values()) { @@ -20425,13 +20410,20 @@ export class OrcaRuntimeService { updatedAt: snapshot.updatedAt }) } - const hookEntries = this.getAgentStatusSnapshotFn?.() ?? [] + const hookEntriesAtInspection = this.getAgentStatusSnapshotFn?.() ?? [] const restoredAgentLiveness = restoredAgentPresenceSupported - ? await this.resolveRestoredAgentLiveness(hookEntries) - : new Map() + ? await this.resolveRestoredAgentLiveness(hookEntriesAtInspection, ptyProcessEvidenceDeadline) + : new Map() + const hookEntries = restoredAgentPresenceSupported + ? (this.getAgentStatusSnapshotFn?.() ?? hookEntriesAtInspection) + : hookEntriesAtInspection for (const entry of hookEntries) { - const liveness = + const livenessResolution = entry.restoredUnconfirmed === true ? restoredAgentLiveness.get(entry.paneKey) : undefined + const liveness = + livenessResolution?.hookIdentity === restoredAgentHookIdentity(entry) + ? livenessResolution.liveness + : undefined // Why: legacy clients ignore restored provenance, while a confirmed shell proves the old agent exited. if ( entry.restoredUnconfirmed === true && @@ -20465,7 +20457,8 @@ export class OrcaRuntimeService { ...(entry.restoredUnconfirmed === true ? { restoredUnconfirmed: true, - agentLiveness: liveness as Exclude + agentLiveness: liveness as Exclude, + restoredExactBinding: livenessResolution?.hasExactBinding === true } : {}) }) @@ -20501,7 +20494,9 @@ export class OrcaRuntimeService { // tabs may exist only remotely), as are rows with no resolvable tabId // (staleness unprovable). Session tabs count as existence: headless // serve has no renderer graph, and session.tabs.list serves them. - continue + if (!src.restoredUnconfirmed || !src.restoredExactBinding) { + continue + } } const worktreeId = mirroredWorktreeId ?? src.worktreeId if (!worktreeId) { @@ -20566,83 +20561,142 @@ export class OrcaRuntimeService { } } - private inspectRestoredAgentPtyOnce(ptyId: string): Promise | null { - const controller = this.ptyController - if (!controller?.inspectProcess) { - return null - } - const pending = this.restoredAgentPtyInspections.get(ptyId) - if (pending?.controller === controller) { - return pending.promise - } - const unavailable: RestoredAgentPtyInspection = { - controller, - foregroundProcess: null, - hasChildProcesses: false, - unavailable: true - } - let entry: RestoredAgentPtyInspectionEntry - const promise = Promise.resolve() - .then(() => controller.inspectProcess!(ptyId)) - .then((inspection) => ({ - controller, - foregroundProcess: inspection.foregroundProcess, - hasChildProcesses: inspection.hasChildProcesses, - unavailable: inspection.unavailable === true - })) - .catch(() => unavailable) - .finally(() => { - if (this.restoredAgentPtyInspections.get(ptyId) === entry) { - this.restoredAgentPtyInspections.delete(ptyId) - } - }) - entry = { controller, promise } - this.restoredAgentPtyInspections.set(ptyId, entry) - return promise - } - private async resolveRestoredAgentLiveness( - entries: readonly AgentStatusIpcPayload[] - ): Promise> { + entries: readonly AgentStatusIpcPayload[], + deadline: number + ): Promise> { const restoredEntries = entries.filter((entry) => entry.restoredUnconfirmed === true) const resolved = await Promise.all( - restoredEntries.map(async (entry): Promise<[string, RestoredAgentLiveness]> => { - const pty = this.getPtyRecordForPaneKey(entry.paneKey) - if (!pty || this.isPtyKnownExited(pty.ptyId)) { - return [entry.paneKey, 'exited'] - } - const inspectionRead = this.inspectRestoredAgentPtyOnce(pty.ptyId) - if (!inspectionRead) { - return [entry.paneKey, 'unverifiable'] - } - const inspection = await inspectionRead - const currentPty = this.getPtyRecordForPaneKey(entry.paneKey) - if ( - inspection.controller !== this.ptyController || - currentPty?.ptyId !== pty.ptyId || - currentPty.incarnationId !== pty.incarnationId || - inspection.unavailable - ) { - return [entry.paneKey, 'unverifiable'] - } - const recognized = recognizeAgentProcess(inspection.foregroundProcess) - if (recognized) { - return [ - entry.paneKey, - isTuiAgent(entry.agentType) && recognized.agent !== entry.agentType ? 'exited' : 'live' - ] - } - return [ + restoredEntries.map(async (entry): Promise<[string, RestoredAgentLivenessResolution]> => { + const hookIdentity = restoredAgentHookIdentity(entry) + const binding = this.getRestoredAgentPtyBinding(entry) + const result = ( + liveness: RestoredAgentLiveness, + hasExactBinding = binding !== null + ): [string, RestoredAgentLivenessResolution] => [ entry.paneKey, - inspection.foregroundProcess !== null && !inspection.hasChildProcesses + { liveness, hookIdentity, hasExactBinding } + ] + if (!binding) { + return result('unverifiable', false) + } + const currentAtStart = this.getPtyRecordForPaneKey(entry.paneKey) + if ( + currentAtStart && + (currentAtStart.ptyId !== binding.ptyId || + (binding.incarnationId !== null && + currentAtStart.incarnationId !== binding.incarnationId)) + ) { + return result('exited') + } + if (currentAtStart && this.isPtyKnownExited(currentAtStart.ptyId)) { + return result('exited') + } + const controller = this.ptyController + if (!controller) { + return result('unverifiable') + } + const evidence = await this.ptyProcessLiveness.inspect({ + source: controller, + ptyId: binding.ptyId, + identity: + binding.incarnationId ?? `runtime:${this.runtimeId}:${binding.lifecycleGeneration}`, + deadline + }) + if ( + controller !== this.ptyController || + !this.isRestoredAgentPtyBindingCurrent(entry, binding) + ) { + return result('unverifiable') + } + if (evidence.status !== 'live') { + return result(evidence.status) + } + if (binding.source === 'persisted') { + return result('unverifiable') + } + const recognized = recognizeAgentProcess(evidence.foregroundProcess) + if (recognized) { + return result( + isTuiAgent(entry.agentType) && recognized.agent !== entry.agentType ? 'exited' : 'live' + ) + } + return result( + evidence.foregroundProcess !== null && evidence.hasChildProcesses === false ? 'exited' : 'unverifiable' - ] + ) }) ) return new Map(resolved) } + private getRestoredAgentPtyBinding(entry: AgentStatusIpcPayload): RestoredAgentPtyBinding | null { + const persisted = this.getPersistedRestoredAgentPtyBinding(entry) + const current = this.getPtyRecordForPaneKey(entry.paneKey) + if (persisted && current) { + return current.ptyId === persisted.ptyId && current.incarnationId === persisted.incarnationId + ? { + ptyId: current.ptyId, + incarnationId: current.incarnationId, + lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId), + source: 'current' + } + : persisted + } + if (current) { + return { + ptyId: current.ptyId, + incarnationId: current.incarnationId, + lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId), + source: 'current' + } + } + return persisted + } + + private getPersistedRestoredAgentPtyBinding( + entry: AgentStatusIpcPayload + ): RestoredAgentPtyBinding | null { + if (!entry.worktreeId) { + return null + } + const session = this.getWorkspaceSessionForWorktree(entry.worktreeId) + for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) { + if ( + binding.paneKey === entry.paneKey && + runtimeWorktreeIdsEqual(binding.worktreeId, entry.worktreeId) + ) { + return { + ptyId, + incarnationId: binding.incarnationId, + lifecycleGeneration: this.getPtyLifecycleGeneration(ptyId), + source: 'persisted' + } + } + } + return null + } + + private isRestoredAgentPtyBindingCurrent( + entry: AgentStatusIpcPayload, + binding: RestoredAgentPtyBinding + ): boolean { + if (this.getPtyLifecycleGeneration(binding.ptyId) !== binding.lifecycleGeneration) { + return false + } + const current = this.getPtyRecordForPaneKey(entry.paneKey) + if (binding.source === 'current') { + return current?.ptyId === binding.ptyId && current.incarnationId === binding.incarnationId + } + const persisted = this.getPersistedRestoredAgentPtyBinding(entry) + return ( + current === null && + persisted?.ptyId === binding.ptyId && + persisted.incarnationId === binding.incarnationId + ) + } + listRepos(): Repo[] { return this.store?.getRepos() ?? [] } @@ -32525,6 +32579,7 @@ export class OrcaRuntimeService { if (state.incarnationId !== undefined) { if (pty.incarnationId && state.incarnationId && pty.incarnationId !== state.incarnationId) { this.invalidatePtyIncarnationHandle(ptyId) + this.advancePtyLifecycleGeneration(ptyId) } pty.incarnationId = state.incarnationId } @@ -32556,6 +32611,9 @@ export class OrcaRuntimeService { pty.paneKey = state.paneKey } if (state.connected !== undefined) { + if (state.connected && !pty.connected) { + this.ptyProcessLiveness.invalidate(ptyId) + } pty.connected = state.connected pty.disconnectedAt = state.connected ? null : (pty.disconnectedAt ?? Date.now()) } @@ -38813,6 +38871,7 @@ export function resolveWorktreeScanCacheTtlMs(repo: Pick { diff --git a/src/main/runtime/pty-process-liveness-broker.test.ts b/src/main/runtime/pty-process-liveness-broker.test.ts new file mode 100644 index 00000000000..aa1f5b8e744 --- /dev/null +++ b/src/main/runtime/pty-process-liveness-broker.test.ts @@ -0,0 +1,134 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { PtyProcessLivenessBroker } from './pty-process-liveness-broker' + +afterEach(() => { + vi.useRealTimers() +}) + +describe('PtyProcessLivenessBroker', () => { + it('bounds a hanging probe and does not fan out another unresolved inspection', async () => { + vi.useFakeTimers() + let resolveInspection!: (value: { + foregroundProcess: string | null + hasChildProcesses: boolean + }) => void + const inspection = new Promise<{ + foregroundProcess: string | null + hasChildProcesses: boolean + }>((resolve) => { + resolveInspection = resolve + }) + const inspectProcess = vi.fn(() => inspection) + const source = { + getForegroundProcess: vi.fn(async () => null), + inspectProcess + } + const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 }) + + const first = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + await vi.advanceTimersByTimeAsync(100) + await expect(first).resolves.toEqual({ + status: 'unverifiable', + reason: 'process inspection timed out' + }) + await expect( + broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + ).resolves.toEqual({ status: 'unverifiable', reason: 'process inspection timed out' }) + expect(inspectProcess).toHaveBeenCalledOnce() + + resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true }) + await vi.runAllTimersAsync() + await expect( + broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + ).resolves.toEqual({ + status: 'live', + foregroundProcess: 'codex', + hasChildProcesses: true + }) + expect(inspectProcess).toHaveBeenCalledOnce() + }) + + it('caches live evidence across sequential clients', async () => { + const inspectProcess = vi.fn(async () => ({ + foregroundProcess: 'codex', + hasChildProcesses: true + })) + const source = { + getForegroundProcess: vi.fn(async () => null), + inspectProcess + } + const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 }) + + await broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + await broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + + expect(inspectProcess).toHaveBeenCalledOnce() + }) + + it('treats only exact terminal_gone evidence as exited', async () => { + const gone = vi.fn(async () => { + throw new Error('terminal_gone') + }) + const unavailable = vi.fn(async () => { + throw new Error('socket_closed') + }) + const onInspectionError = vi.fn() + const broker = new PtyProcessLivenessBroker({ timeoutMs: 100, onInspectionError }) + + await expect( + broker.inspect({ + source: { getForegroundProcess: vi.fn(async () => null), inspectProcess: gone }, + ptyId: 'pty-gone', + identity: 'incarnation-a' + }) + ).resolves.toEqual({ status: 'exited' }) + await expect( + broker.inspect({ + source: { getForegroundProcess: vi.fn(async () => null), inspectProcess: unavailable }, + ptyId: 'pty-unavailable', + identity: 'incarnation-b' + }) + ).resolves.toEqual({ status: 'unverifiable', reason: 'socket_closed' }) + expect(onInspectionError).toHaveBeenCalledExactlyOnceWith( + 'pty-unavailable', + expect.objectContaining({ message: 'socket_closed' }) + ) + }) + + it('does not let an old incarnation populate the replacement cache entry', async () => { + let resolveOld!: (value: { foregroundProcess: string; hasChildProcesses: boolean }) => void + const oldInspection = new Promise<{ + foregroundProcess: string + hasChildProcesses: boolean + }>((resolve) => { + resolveOld = resolve + }) + const inspectProcess = vi + .fn() + .mockReturnValueOnce(oldInspection) + .mockResolvedValueOnce({ foregroundProcess: 'zsh', hasChildProcesses: false }) + const source = { + getForegroundProcess: vi.fn(async () => null), + inspectProcess + } + const broker = new PtyProcessLivenessBroker({ timeoutMs: 1_000 }) + + const oldRead = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + const replacementRead = broker.inspect({ + source, + ptyId: 'pty-1', + identity: 'incarnation-b' + }) + resolveOld({ foregroundProcess: 'codex', hasChildProcesses: true }) + + await expect(oldRead).resolves.toMatchObject({ status: 'live', foregroundProcess: 'codex' }) + await expect(replacementRead).resolves.toMatchObject({ + status: 'live', + foregroundProcess: 'zsh' + }) + await expect( + broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-b' }) + ).resolves.toMatchObject({ status: 'live', foregroundProcess: 'zsh' }) + expect(inspectProcess).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/runtime/pty-process-liveness-broker.ts b/src/main/runtime/pty-process-liveness-broker.ts new file mode 100644 index 00000000000..ec6a6d38268 --- /dev/null +++ b/src/main/runtime/pty-process-liveness-broker.ts @@ -0,0 +1,250 @@ +export type PtyProcessInspectionSource = { + getForegroundProcess(ptyId: string): Promise + inspectProcess?: (ptyId: string) => Promise<{ + foregroundProcess: string | null + hasChildProcesses: boolean + unavailable?: true + }> + hasChildProcesses?: (ptyId: string) => Promise +} + +export type PtyProcessLivenessEvidence = + | { + status: 'live' + foregroundProcess: string | null + hasChildProcesses: boolean | null + } + | { status: 'unverifiable'; reason: string } + | { status: 'exited' } + +type PtyProcessEvidenceEntry = { + source: PtyProcessInspectionSource + identity: string + freshness: number + failureCount: number + evidence: PtyProcessLivenessEvidence | null + expiresAt: number + timedOut: boolean + probe: Promise | null +} + +export type PtyProcessLivenessBrokerOptions = { + timeoutMs: number + liveTtlMs?: number + unavailableBackoffBaseMs?: number + unavailableBackoffMaxMs?: number + now?: () => number + onInspectionError?: (ptyId: string, error: unknown) => void +} + +const DEFAULT_LIVE_TTL_MS = 10_000 +const DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS = 3_000 +const DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS = 30_000 + +export class PtyProcessLivenessBroker { + private readonly entries = new Map() + private readonly now: () => number + + constructor(private readonly options: PtyProcessLivenessBrokerOptions) { + this.now = options.now ?? Date.now + } + + inspect(args: { + source: PtyProcessInspectionSource + ptyId: string + identity: string + freshness?: number + reuseSettled?: boolean + deadline?: number + }): Promise { + const freshness = args.freshness ?? 0 + const waitMs = Math.max( + 0, + Math.min( + this.options.timeoutMs, + args.deadline === undefined ? this.options.timeoutMs : args.deadline - this.now() + ) + ) + const existing = this.entries.get(args.ptyId) + if ( + existing?.source === args.source && + existing.identity === args.identity && + existing.probe && + existing.freshness < freshness + ) { + if (waitMs === 0) { + return Promise.resolve({ + status: 'unverifiable', + reason: 'process inspection timed out' + }) + } + return this.waitForProbe(args.ptyId, existing, waitMs).then(() => + existing.probe + ? { status: 'unverifiable', reason: 'process inspection timed out' } + : this.inspect(args) + ) + } + if ( + existing?.source === args.source && + existing.identity === args.identity && + existing.freshness >= freshness + ) { + if (existing.probe) { + return existing.timedOut + ? Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' }) + : waitMs === 0 + ? Promise.resolve({ + status: 'unverifiable', + reason: 'process inspection timed out' + }) + : this.waitForProbe(args.ptyId, existing, waitMs) + } + if (args.reuseSettled !== false && existing.evidence && existing.expiresAt > this.now()) { + return Promise.resolve(existing.evidence) + } + } + if (waitMs === 0) { + return Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' }) + } + + const failureCount = + existing?.source === args.source && existing.identity === args.identity + ? existing.failureCount + : 0 + const entry: PtyProcessEvidenceEntry = { + source: args.source, + identity: args.identity, + freshness, + failureCount, + evidence: null, + expiresAt: 0, + timedOut: false, + probe: null + } + const probe = this.runProbe(args.source, args.ptyId) + .then((evidence) => { + if (this.entries.get(args.ptyId) !== entry) { + return evidence + } + entry.probe = null + entry.timedOut = false + entry.evidence = evidence + if (evidence.status === 'unverifiable') { + entry.failureCount += 1 + entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount) + } else if (evidence.status === 'live') { + entry.failureCount = 0 + entry.expiresAt = this.now() + (this.options.liveTtlMs ?? DEFAULT_LIVE_TTL_MS) + } else { + entry.failureCount = 0 + entry.expiresAt = Number.POSITIVE_INFINITY + } + return evidence + }) + .catch((error): PtyProcessLivenessEvidence => { + if (this.entries.get(args.ptyId) === entry) { + entry.probe = null + entry.timedOut = false + entry.failureCount += 1 + entry.evidence = { status: 'unverifiable', reason: describeError(error) } + entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount) + } + this.options.onInspectionError?.(args.ptyId, error) + return { status: 'unverifiable', reason: describeError(error) } + }) + entry.probe = probe + this.entries.set(args.ptyId, entry) + return this.waitForProbe(args.ptyId, entry, waitMs) + } + + invalidate(ptyId: string): void { + this.entries.delete(ptyId) + } + + invalidateAll(): void { + this.entries.clear() + } + + getPendingCount(): number { + let count = 0 + for (const entry of this.entries.values()) { + if (entry.probe) { + count += 1 + } + } + return count + } + + private async runProbe( + source: PtyProcessInspectionSource, + ptyId: string + ): Promise { + try { + if (source.inspectProcess) { + const inspection = await source.inspectProcess(ptyId) + return inspection.unavailable + ? { status: 'unverifiable', reason: 'process inspection unavailable' } + : { + status: 'live', + foregroundProcess: inspection.foregroundProcess, + hasChildProcesses: inspection.hasChildProcesses + } + } + const foregroundProcess = await source.getForegroundProcess(ptyId) + return { + status: 'live', + foregroundProcess, + hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null + } + } catch (error) { + if (isTerminalGoneError(error)) { + return { status: 'exited' } + } + throw error + } + } + + private waitForProbe( + ptyId: string, + entry: PtyProcessEvidenceEntry, + timeoutMs: number + ): Promise { + const probe = entry.probe + if (!probe) { + return Promise.resolve( + entry.evidence ?? { status: 'unverifiable', reason: 'process inspection unavailable' } + ) + } + return new Promise((resolve) => { + const timeout = setTimeout(() => { + if (this.entries.get(ptyId) === entry && entry.probe === probe) { + entry.timedOut = true + } + resolve({ status: 'unverifiable', reason: 'process inspection timed out' }) + }, timeoutMs) + void probe.then((evidence) => { + clearTimeout(timeout) + resolve(evidence) + }) + }) + } + + private unavailableBackoffMs(failureCount: number): number { + const base = this.options.unavailableBackoffBaseMs ?? DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS + const max = this.options.unavailableBackoffMaxMs ?? DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS + return Math.min(max, base * 2 ** Math.max(0, failureCount - 1)) + } +} + +function isTerminalGoneError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error) + const code = + error && typeof error === 'object' && 'code' in error + ? String((error as { code?: unknown }).code) + : '' + return message === 'terminal_gone' || code === 'terminal_gone' +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error) +}