From 125784bcea17b327ea1ace154df7911f1ff54dff Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Sun, 23 Aug 2026 20:43:32 -0400 Subject: [PATCH] fix(runtime): bound restored agent authority recovery --- config/reliability-gates.jsonc | 25 +- src/main/runtime/orca-runtime.test.ts | 170 ++++++++- src/main/runtime/orca-runtime.ts | 350 ++++++++++++------ src/main/runtime/pty-process-inspection.ts | 60 +++ .../pty-process-liveness-broker.test.ts | 74 ++++ .../runtime/pty-process-liveness-broker.ts | 140 ++++--- .../restored-agent-authority-resolver.test.ts | 114 +++++- .../restored-agent-authority-resolver.ts | 10 +- 8 files changed, 753 insertions(+), 190 deletions(-) create mode 100644 src/main/runtime/pty-process-inspection.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index abfedf244ea..22e96419047 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10272,7 +10272,7 @@ "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["local", "daemon", "ssh"], - "coverageNotes": "A deterministic service-state-machine oracle now models a current-contract worker and coordinator whose renderer graph identities disappear across an app/runtime update. It exercises the production verifier with restored PTY and hydrated hook commitments, proves authenticated completion replay across a fresh runtime, explicit takeover, ordinary mail routing, remote-attachment process fencing, retained Task/Dispatch/terminal identity, and unchanged fixture marker bytes, and rejects foreign pane evidence. Worktree inventory coverage now capability-gates restored presence, resolves each exact PTY on the execution host as live, unverifiable, or exited, keeps live/unverifiable identities visible without reviving stale activity, preserves byte-compatible legacy suppression, and keeps completed shell-only rows retired. Other deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, creator pane/process/Run-generation fencing, indexed retained-Run lookup scaling, the SSH in-process CLI fallback, exact legacy Task/Dispatch/report-receipt verification, JSON rejection envelopes, and capability-negotiated Run-home settlement. The federation oracle drops the first terminal ACK, restarts Run-home twice, verifies replay through the existing polling path and the persisted acknowledgment watermark, races identical reports, accepts preexisting same-outcome ACK batches, rejects conflicting batches, and covers protocol v1/v2 completion across independently updated peers. A migration unit upgrades a v26 database with a zeroed watermark without replaying older migrations. Three isolated macOS Electron journeys launch fake Codex workers through the real RPC path. Distinct installed A/B artifacts plus live SSH, WSL, folder, Linux, and Windows cutover journeys remain explicit gaps.", + "coverageNotes": "A deterministic service-state-machine oracle now models a current-contract worker and coordinator whose renderer graph identities disappear across an app/runtime update. It exercises the production verifier with restored PTY and hydrated hook commitments, proves authenticated completion replay across a fresh runtime, explicit takeover, ordinary mail routing, remote-attachment process fencing, retained Task/Dispatch/terminal identity, and unchanged fixture marker bytes, and rejects foreign pane evidence. Worktree inventory coverage now capability-gates restored presence, resolves each exact PTY on the execution host as live, unverifiable, or exited, keeps live/unverifiable identities visible without reviving stale activity, preserves byte-compatible legacy suppression, and keeps completed shell-only rows retired. Persisted surface identity must seed first authority, request-wide indexes preserve ambiguous-binding refusal without per-row PTY scans, and shared inspection admits at most eight unresolved provider probes while restored-only cache entries are pruned when hooks disappear. Other deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, creator pane/process/Run-generation fencing, indexed retained-Run lookup scaling, the SSH in-process CLI fallback, exact legacy Task/Dispatch/report-receipt verification, JSON rejection envelopes, and capability-negotiated Run-home settlement. The federation oracle drops the first terminal ACK, restarts Run-home twice, verifies replay through the existing polling path and the persisted acknowledgment watermark, races identical reports, accepts preexisting same-outcome ACK batches, rejects conflicting batches, and covers protocol v1/v2 completion across independently updated peers. A migration unit upgrades a v26 database with a zeroed watermark without replaying older migrations. Three isolated macOS Electron journeys launch fake Codex workers through the real RPC path. Distinct installed A/B artifacts plus live SSH, WSL, folder, Linux, and Windows cutover journeys remain explicit gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/11107#discussion_r3663321387", "https://github.com/stablyai/orca/issues/13858", @@ -10355,11 +10355,21 @@ "file": "src/main/runtime/restored-agent-authority-resolver.test.ts", "assertions": [ "one restored hook cannot borrow a replacement PTY incarnation or another execution host", + "current-only surface identity cannot establish first authority; exact persisted identity seeds it before matching current state may continue", "conflicting current and persisted bindings remain unverifiable", "incarnation identity survives local lifecycle-generation changes", + "tagged fallback generations cannot collide with generation-shaped incarnation ids, and optional handles do not redefine process equality", "retired hook commitments are released without unbounded growth" ] }, + { + "file": "src/main/runtime/pty-process-liveness-broker.test.ts", + "assertions": [ + "cross-PTY hanging probes remain below the broker-wide admission cap", + "retired restored-agent entries are pruned without deleting evidence shared by other process consumers", + "per-identity timeouts, settlement, TTL reuse, backoff, exact absence, and owning-inventory reconciliation remain single-flight" + ] + }, { "file": "src/main/runtime/orca-runtime.test.ts", "assertions": [ @@ -10374,7 +10384,8 @@ "terminal_gone is the only inspection rejection that retires the exact restored row", "fresh exact inventory cannot be overwritten by older cached terminal_gone evidence after later contact loss", "slow foreground inspection cannot manufacture agent exit or certify a replacement incarnation", - "sequential inventory clients reuse bounded process evidence instead of polling each restored row" + "sequential inventory clients reuse bounded process evidence instead of polling each restored row", + "multi-row catalogs build two host-session authority indexes per request and inspect at most four restored rows concurrently" ] }, { @@ -10515,8 +10526,8 @@ "platform": "macos", "command": "pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1", "result": "passed", - "durationSeconds": 49.8, - "summary": "Both legacy- and current-contract isolated restart journeys kept the original live PID/PTTY/incarnation, visible inactive tab, writable ACK output, and active Dispatch, while capable worktree.ps retained the exact agent row as unverifiable because the fresh desktop could not inspect the daemon-owned process. Disabling only restored-presence negotiation made the byte-identical journey fail with agent undefined while all preceding liveness, tab, and Dispatch checks stayed green." + "durationSeconds": 54.8, + "summary": "Both legacy- and current-contract isolated restart journeys kept the original live PID/PTTY/incarnation, visible inactive tab exactly once, writable ACK output, and active Dispatch, while capable worktree.ps retained the exact agent row as unverifiable because the fresh desktop could not inspect the daemon-owned process. Disabling only restored-presence negotiation made the byte-identical journey fail with agent undefined while all preceding liveness, tab, and Dispatch checks stayed green." }, { "date": "2026-08-23", @@ -10524,8 +10535,8 @@ "platform": "macos", "command": "pnpm exec vitest run --config config/vitest.config.ts --fileParallelism=false --maxWorkers=1 src/main/runtime/restored-agent-authority-resolver.test.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts", "result": "passed", - "durationSeconds": 49.44, - "summary": "Ten focused runtime and wire files passed 1,292 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, three-poll fresh-presence reconciliation, cold handle-qualified contact loss, incarnation/host/hook replacement fencing, slow foreground completion, callback isolation, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, encrypted desktop/CLI capability advertisement, and cross-version terminal wire compatibility. Full mobile typecheck and 3,674 tests passed separately." + "durationSeconds": 27.06, + "summary": "Ten focused runtime and wire files passed 1,301 tests with one skip, including persisted-seeded first authority, replacement-current refusal, tagged incarnation identity, handle-remint continuity, two request-wide authority indexes, four-wide catalog inspection, eight-wide global hanging-probe admission, consumer-scoped cache pruning, bounded inspection timeout, exact terminal_gone absence, three-poll fresh-presence reconciliation, cold handle-qualified contact loss, slow foreground completion, callback isolation, live/unverifiable/exited/hidden-tab/legacy projection, encrypted desktop/CLI capability advertisement, and cross-version terminal wire compatibility. Full mobile typecheck and 3,674 tests passed separately." }, { "date": "2026-08-13", @@ -10704,7 +10715,7 @@ }, "performanceBudget": { "required": true, - "evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, a hung provider retains at most one unresolved probe per identity, and every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path." + "evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, and at most eight unresolved provider probes can exist across PTYs even when their promises never settle. Restored projection builds one O(PTYs + persisted surfaces + restored rows) authority index before inspection and one after it, maps catalog work four-wide, and prunes entries owned only by disappeared restored hooks without pruning shared foreground evidence. Every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path." }, "promotionCriteria": [ "Collect 100 consecutive passes or 14 days of stable CI history on macOS, Linux, and Windows.", diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 8b4795a6682..476a496ffb5 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -1049,6 +1049,7 @@ const TEST_FOLDER_WORKSPACE_KEY = `folder:${TEST_FOLDER_WORKSPACE_ID}` const TEST_FOLDER_WORKSPACE_PATH = '/tmp/platform' const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ const HEADLESS_LEAF_ID = '11111111-1111-4111-8111-111111111111' +const RESTORED_AGENT_INSPECTION_TEST_CONCURRENCY = 4 const HEADLESS_SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222' const HEADLESS_THIRD_LEAF_ID = '33333333-3333-4333-8333-333333333333' const RESTORED_AUTHORITY_TOKEN = 'restored-authority-secret' @@ -1693,6 +1694,37 @@ function makeWorkspaceSessionWithHeadlessTerminal( } } +function makePersistedRestoredAgentSession(args: { + tabId: string + leafId: string + ptyId: string + incarnationId: string +}): WorkspaceSessionState { + const paneKey = makePaneKey(args.tabId, args.leafId) + return makeWorkspaceSessionWithHeadlessTerminal({ + activeTabId: args.tabId, + activeTabIdByWorktree: { [TEST_WORKTREE_ID]: args.tabId }, + tabsByWorktree: { + [TEST_WORKTREE_ID]: [ + { + id: args.tabId, + ptyId: args.ptyId, + worktreeId: TEST_WORKTREE_ID, + title: 'Persisted Agent', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + [args.tabId]: makeHeadlessTerminalLayout({ [args.leafId]: args.ptyId }) + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: args.incarnationId } + }) +} + computeWorktreePathMock.mockImplementation( ( sanitizedName: string, @@ -37991,6 +38023,78 @@ describe('OrcaRuntimeService', () => { expect(inspectProcess).not.toHaveBeenCalled() }) + it('indexes restored authority once per snapshot and bounds request inspection fanout', async () => { + const rowCount = 12 + const tabs = Array.from({ length: rowCount }, (_, index) => ({ + id: `tab-${index}`, + ptyId: `persisted-pty-${index}`, + worktreeId: TEST_WORKTREE_ID, + title: `Agent ${index}`, + customTitle: null, + color: null, + sortOrder: index, + createdAt: index + 1 + })) + const terminalLayoutsByTabId = Object.fromEntries( + tabs.map((tab) => [tab.id, makeHeadlessTerminalLayout({ [HEADLESS_LEAF_ID]: tab.ptyId })]) + ) + const terminalPtyIncarnationsByPaneKey = Object.fromEntries( + tabs.map((tab, index) => [makePaneKey(tab.id, HEADLESS_LEAF_ID), `incarnation-${index}`]) + ) + const session = makeWorkspaceSessionWithHeadlessTerminal({ + tabsByWorktree: { [TEST_WORKTREE_ID]: tabs }, + terminalLayoutsByTabId, + terminalPtyIncarnationsByPaneKey + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const getWorkspaceSession = vi.fn(runtimeStore.getWorkspaceSession) + let activeInspections = 0 + let maxActiveInspections = 0 + const inspectProcess = vi.fn(async () => { + activeInspections += 1 + maxActiveInspections = Math.max(maxActiveInspections, activeInspections) + await Promise.resolve() + activeInspections -= 1 + return { foregroundProcess: 'codex', hasChildProcesses: true } + }) + const now = Date.now() + const runtime = new OrcaRuntimeService( + { ...runtimeStore, getWorkspaceSession } as never, + undefined, + { + getAgentStatusSnapshot: () => + tabs.map((tab, index) => ({ + paneKey: makePaneKey(tab.id, HEADLESS_LEAF_ID), + worktreeId: TEST_WORKTREE_ID, + tabId: tab.id, + state: 'working' as const, + prompt: `restored agent ${index}`, + agentType: 'codex', + connectionId: null, + receivedAt: now + index, + stateStartedAt: now - rowCount + index, + restoredUnconfirmed: true + })) + } + ) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: vi.fn(async () => 'codex'), + inspectProcess, + listProcesses: vi.fn(async () => []) + }) + + const { worktrees } = await runtime.getWorktreePs(undefined, true, true) + + expect( + worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents + ).toHaveLength(rowCount) + expect(inspectProcess).toHaveBeenCalledTimes(rowCount) + expect(maxActiveInspections).toBe(RESTORED_AGENT_INSPECTION_TEST_CONCURRENCY) + expect(getWorkspaceSession.mock.calls.length).toBeLessThanOrEqual(4) + }) + it('omits a cold restored row only after terminal_gone confirms exact absence', async () => { const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID) const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' @@ -38085,6 +38189,14 @@ describe('OrcaRuntimeService', () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession( + makePersistedRestoredAgentSession({ + tabId: 'tab-1', + leafId, + ptyId: 'pty-1', + incarnationId + }) + ) const now = Date.now() const inspectProcess = vi.fn(async () => { throw new Error('terminal_gone') @@ -38102,7 +38214,7 @@ describe('OrcaRuntimeService', () => { } ]) .mockRejectedValueOnce(new Error('socket_closed')) - const runtime = new OrcaRuntimeService(store, undefined, { + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { getAgentStatusSnapshot: () => [ { paneKey, @@ -38183,12 +38295,21 @@ describe('OrcaRuntimeService', () => { it('projects a restored agent after the execution host confirms its exact PTY is live', async () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` + const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession( + makePersistedRestoredAgentSession({ + tabId: 'tab-1', + leafId, + ptyId: 'pty-1', + incarnationId + }) + ) const now = Date.now() const inspectProcess = vi.fn(async () => ({ foregroundProcess: 'node /opt/codex/bin/codex', hasChildProcesses: true })) - const runtime = new OrcaRuntimeService(store, undefined, { + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { getAgentStatusSnapshot: () => [ { paneKey, @@ -38209,7 +38330,9 @@ describe('OrcaRuntimeService', () => { kill: vi.fn(() => true), getForegroundProcess: vi.fn(async () => 'node /opt/codex/bin/codex'), inspectProcess, - listProcesses: vi.fn(async () => [{ id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'Codex' }]) + listProcesses: vi.fn(async () => [ + { id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'Codex', incarnationId } + ]) }) runtime.attachWindow(1) runtime.syncWindowGraph(1, { @@ -38403,6 +38526,15 @@ describe('OrcaRuntimeService', () => { it('fences a restored-agent inspection across PTY reincarnation', async () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` + const incarnationId = 'incarnation-a' + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession( + makePersistedRestoredAgentSession({ + tabId: 'tab-1', + leafId, + ptyId: 'pty-1', + incarnationId + }) + ) const now = Date.now() let resolveInspection!: (value: { foregroundProcess: string @@ -38414,7 +38546,7 @@ describe('OrcaRuntimeService', () => { resolveInspection = resolve }) ) - const runtime = new OrcaRuntimeService(store, undefined, { + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { getAgentStatusSnapshot: () => [ { paneKey, @@ -38440,7 +38572,7 @@ describe('OrcaRuntimeService', () => { runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, { tabId: 'tab-1', leafId, - incarnationId: 'incarnation-a' + incarnationId }) runtime.attachWindow(1) runtime.syncWindowGraph(1, { @@ -38494,6 +38626,15 @@ describe('OrcaRuntimeService', () => { it('does not publish a restored hook snapshot replaced during inspection', async () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` + const incarnationId = 'incarnation-a' + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession( + makePersistedRestoredAgentSession({ + tabId: 'tab-1', + leafId, + ptyId: 'pty-1', + incarnationId + }) + ) const now = Date.now() let rows: AgentStatusIpcPayload[] = [ { @@ -38519,7 +38660,7 @@ describe('OrcaRuntimeService', () => { resolveInspection = resolve }) ) - const runtime = new OrcaRuntimeService(store, undefined, { + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { getAgentStatusSnapshot: () => rows }) runtime.setPtyController({ @@ -38532,7 +38673,7 @@ describe('OrcaRuntimeService', () => { runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, { tabId: 'tab-1', leafId, - incarnationId: 'incarnation-a' + incarnationId }) runtime.attachWindow(1) runtime.syncWindowGraph(1, { @@ -38570,12 +38711,21 @@ describe('OrcaRuntimeService', () => { it('omits an exited restored agent after the exact PTY is confirmed shell-only', async () => { const leafId = '33333333-3333-4333-8333-333333333333' const paneKey = `tab-1:${leafId}` + const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession( + makePersistedRestoredAgentSession({ + tabId: 'tab-1', + leafId, + ptyId: 'pty-1', + incarnationId + }) + ) const now = Date.now() const inspectProcess = vi.fn(async () => ({ foregroundProcess: 'zsh', hasChildProcesses: false })) - const runtime = new OrcaRuntimeService(store, undefined, { + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { getAgentStatusSnapshot: () => [ { paneKey, @@ -38596,7 +38746,9 @@ describe('OrcaRuntimeService', () => { kill: vi.fn(() => true), getForegroundProcess: vi.fn(async () => 'zsh'), inspectProcess, - listProcesses: vi.fn(async () => [{ id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'zsh' }]) + listProcesses: vi.fn(async () => [ + { id: 'pty-1', cwd: TEST_WORKTREE_PATH, title: 'zsh', incarnationId } + ]) }) runtime.attachWindow(1) runtime.syncWindowGraph(1, { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 162b848b04f..58690997665 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -660,7 +660,11 @@ import type { RuntimeBrowserCommands } from './orca-runtime-browser' import { createRuntimeBrowserCommands } from './runtime-browser-commands-factory' import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency' import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity' -import { PtyProcessLivenessBroker } from './pty-process-liveness-broker' +import { + PtyProcessLivenessBroker, + type PtyProcessLivenessEvidence +} from './pty-process-liveness-broker' +import { mapWithConcurrency } from '../../shared/map-with-concurrency' import { RestoredAgentAuthorityResolver, sameRestoredAgentProcessAuthority, @@ -1998,6 +2002,19 @@ type PtyControllerInventory = Readonly<{ queriedHostIds: ReadonlySet }> +type PersistedRestoredAgentSurface = Readonly<{ + ptyId: string + worktreeId: string + paneKey: string + incarnationId: string +}> + +type RestoredAgentAuthoritySnapshot = Readonly<{ + currentPtyByPaneKey: ReadonlyMap + persistedSurfaceByKey: ReadonlyMap + inventory: PtyControllerInventory | null +}> + type WorktreeStartupDraftPaste = { agent: TuiAgent content: string @@ -19090,9 +19107,10 @@ export class OrcaRuntimeService { } private getPtyProcessEvidenceIdentity(ptyId: string): string { - return ( - this.ptysById.get(ptyId)?.incarnationId ?? - `runtime:${this.runtimeId}:${this.getPtyLifecycleGeneration(ptyId)}` + return makePtyProcessEvidenceIdentity( + this.ptysById.get(ptyId)?.incarnationId ?? null, + this.runtimeId, + this.getPtyLifecycleGeneration(ptyId) ) } @@ -20591,14 +20609,76 @@ export class OrcaRuntimeService { freshPtyInventory: PtyControllerInventory | null ): Promise> { const restoredEntries = entries.filter((entry) => entry.restoredUnconfirmed === true) - this.restoredAgentAuthority.retain( - new Set(restoredEntries.map((entry) => restoredAgentHookIdentity(entry))) + const retainedHookIdentities = new Set( + restoredEntries.map((entry) => restoredAgentHookIdentity(entry)) ) - const resolved = await Promise.all( - restoredEntries.map(async (entry): Promise<[string, RestoredAgentLivenessResolution]> => { - const hookIdentity = restoredAgentHookIdentity(entry) - const authority = this.getRestoredAgentAuthority(entry, freshPtyInventory) + this.restoredAgentAuthority.retain(retainedHookIdentities) + const hooks = restoredEntries.map((entry) => ({ + entry, + hookIdentity: restoredAgentHookIdentity(entry), + hook: this.getRestoredAgentAuthorityHook(entry) + })) + const initialSnapshot = this.buildRestoredAgentAuthoritySnapshot( + restoredEntries, + freshPtyInventory + ) + const initial = hooks.map((item) => ({ + ...item, + authority: item.hook + ? this.getRestoredAgentAuthority(item.hook, initialSnapshot) + : { binding: null, hasExactBinding: false } + })) + this.ptyProcessLiveness.retainConsumerEvidence( + RESTORED_AGENT_PROCESS_EVIDENCE_CONSUMER, + initial.flatMap(({ authority }) => { const binding = authority.binding + return binding + ? [{ ptyId: binding.ptyId, identity: this.getRestoredAgentEvidenceIdentity(binding) }] + : [] + }) + ) + const controller = this.ptyController + const observed = await mapWithConcurrency( + initial, + RESTORED_AGENT_INSPECTION_CONCURRENCY, + async ( + item + ): Promise<{ + item: (typeof initial)[number] + evidence: PtyProcessLivenessEvidence | null + settledLiveness: RestoredAgentLiveness | null + }> => { + const binding = item.authority.binding + if (!binding) { + return { item, evidence: null, settledLiveness: 'unverifiable' } + } + if (this.isPtyKnownExited(binding.ptyId)) { + return { item, evidence: null, settledLiveness: 'exited' } + } + if (!controller) { + return { item, evidence: null, settledLiveness: 'unverifiable' } + } + const evidence = await this.ptyProcessLiveness.inspect({ + source: controller, + ptyId: binding.ptyId, + identity: this.getRestoredAgentEvidenceIdentity(binding), + deadline, + owningInventoryObservedPty: this.inventoryProvesRestoredBindingPresent( + freshPtyInventory, + binding + ), + consumerId: RESTORED_AGENT_PROCESS_EVIDENCE_CONSUMER + }) + return { item, evidence, settledLiveness: null } + } + ) + const validationSnapshot = this.buildRestoredAgentAuthoritySnapshot( + restoredEntries, + freshPtyInventory + ) + return new Map( + observed.map(({ item, evidence, settledLiveness }) => { + const { entry, hookIdentity, authority, hook } = item const result = ( liveness: RestoredAgentLiveness, hasExactBinding = authority.hasExactBinding @@ -20606,37 +20686,23 @@ export class OrcaRuntimeService { entry.paneKey, { liveness, hookIdentity, hasExactBinding } ] - if (!binding) { - return result('unverifiable') + if (settledLiveness) { + return result(settledLiveness) } - if (this.isPtyKnownExited(binding.ptyId)) { - return result('exited') - } - const controller = this.ptyController - if (!controller) { - return result('unverifiable') - } - const evidence = await this.ptyProcessLiveness.inspect({ - source: controller, - ptyId: binding.ptyId, - identity: - binding.incarnationId ?? `runtime:${this.runtimeId}:${binding.lifecycleGeneration}`, - deadline, - owningInventoryObservedPty: this.inventoryProvesRestoredBindingPresent( - freshPtyInventory, - binding - ) - }) - const currentAuthority = this.getRestoredAgentAuthority(entry, freshPtyInventory).binding + const binding = authority.binding + const currentAuthority = hook + ? this.getRestoredAgentAuthority(hook, validationSnapshot).binding + : null if ( controller !== this.ptyController || + !binding || !currentAuthority || !sameRestoredAgentProcessAuthority(currentAuthority, binding) ) { return result('unverifiable') } - if (evidence.status !== 'live') { - return result(evidence.status) + if (!evidence || evidence.status !== 'live') { + return result(evidence?.status ?? 'unverifiable') } if (binding.source === 'persisted') { return result('unverifiable') @@ -20654,85 +20720,72 @@ export class OrcaRuntimeService { ) }) ) - return new Map(resolved) } private getRestoredAgentAuthority( - entry: AgentStatusIpcPayload, - inventory: PtyControllerInventory | null + hook: RestoredAgentAuthorityHook, + snapshot: RestoredAgentAuthoritySnapshot ) { - const hook = this.getRestoredAgentAuthorityHook(entry) - if (!hook) { - return { binding: null, hasExactBinding: false } - } return this.restoredAgentAuthority.resolve({ hook, - current: this.getCurrentRestoredAgentBinding(entry, hook, inventory), - persisted: this.getPersistedRestoredAgentPtyBinding(entry, hook, inventory) + current: this.getCurrentRestoredAgentBinding(hook, snapshot), + persisted: this.getPersistedRestoredAgentPtyBinding(hook, snapshot) }) } private getPersistedRestoredAgentPtyBinding( - entry: AgentStatusIpcPayload, hook: RestoredAgentAuthorityHook, - inventory: PtyControllerInventory | null + snapshot: RestoredAgentAuthoritySnapshot ): RestoredAgentAuthorityBinding | null { - if (!entry.worktreeId) { + const binding = snapshot.persistedSurfaceByKey.get( + restoredAgentSurfaceKey(hook.hostKey, hook.worktreeKey, hook.paneKey) + ) + if (!binding) { return null } - const session = this.getWorkspaceSessionForWorktree(entry.worktreeId) - for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) { - if ( - binding.paneKey === entry.paneKey && - runtimeWorktreeIdsEqual(binding.worktreeId, entry.worktreeId) - ) { - const controllerIdentity = inventory?.terminalIdentityByPtyId.get(ptyId) - const encodedHost = getPtyExecutionHost(ptyId) - if ( - encodedHost === 'foreign' || - (encodedHost !== null && encodedHost !== hook.hostKey) || - (hook.hostKey.startsWith('wsl:') && - controllerIdentity?.wslDistro?.toLowerCase() !== hook.hostKey.slice(4)) - ) { - return null - } - return { - ptyId, - incarnationId: binding.incarnationId, - lifecycleGeneration: this.getPtyLifecycleGeneration(ptyId), - source: 'persisted', - paneKey: binding.paneKey, - worktreeKey: hook.worktreeKey, - hostKey: hook.hostKey, - ...((controllerIdentity?.handle ?? hook.terminalHandle) - ? { terminalHandle: controllerIdentity?.handle ?? hook.terminalHandle } - : {}) - } - } + const controllerIdentity = snapshot.inventory?.terminalIdentityByPtyId.get(binding.ptyId) + const encodedHost = getPtyExecutionHost(binding.ptyId) + if ( + encodedHost === 'foreign' || + (encodedHost !== null && encodedHost !== hook.hostKey) || + (hook.hostKey.startsWith('wsl:') && + controllerIdentity?.wslDistro?.toLowerCase() !== hook.hostKey.slice(4)) + ) { + return null + } + return { + ptyId: binding.ptyId, + incarnationId: binding.incarnationId, + lifecycleGeneration: this.getPtyLifecycleGeneration(binding.ptyId), + source: 'persisted', + paneKey: binding.paneKey, + worktreeKey: hook.worktreeKey, + hostKey: hook.hostKey, + ...((controllerIdentity?.handle ?? hook.terminalHandle) + ? { terminalHandle: controllerIdentity?.handle ?? hook.terminalHandle } + : {}) } - return null } private getCurrentRestoredAgentBinding( - entry: AgentStatusIpcPayload, hook: RestoredAgentAuthorityHook, - inventory: PtyControllerInventory | null + snapshot: RestoredAgentAuthoritySnapshot ): RestoredAgentAuthorityBinding | null { - const current = this.getExactPtyRecordForPaneKey(entry.paneKey) + const current = snapshot.currentPtyByPaneKey.get(hook.paneKey) if ( !current || - !runtimeWorktreeIdsEqual(current.worktreeId, entry.worktreeId ?? '') || + runtimeWorktreeIdentityKey(current.worktreeId) !== hook.worktreeKey || !this.ptyMatchesRestoredAgentHost(current, hook.hostKey) ) { return null } - const controllerIdentity = inventory?.terminalIdentityByPtyId.get(current.ptyId) + const controllerIdentity = snapshot.inventory?.terminalIdentityByPtyId.get(current.ptyId) return { ptyId: current.ptyId, incarnationId: current.incarnationId, lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId), source: 'current', - paneKey: entry.paneKey, + paneKey: hook.paneKey, worktreeKey: hook.worktreeKey, hostKey: hook.hostKey, ...((controllerIdentity?.handle ?? @@ -20748,6 +20801,100 @@ export class OrcaRuntimeService { } } + private buildRestoredAgentAuthoritySnapshot( + entries: readonly AgentStatusIpcPayload[], + inventory: PtyControllerInventory | null + ): RestoredAgentAuthoritySnapshot { + const requestedPaneKeys = new Set(entries.map((entry) => entry.paneKey)) + const currentCandidates = new Map>() + const addCurrent = (paneKey: string, pty: RuntimePtyWorktreeRecord | undefined): void => { + if (!pty || !requestedPaneKeys.has(paneKey)) { + return + } + const candidates = currentCandidates.get(paneKey) ?? new Set() + candidates.add(pty) + currentCandidates.set(paneKey, candidates) + } + for (const paneKey of requestedPaneKeys) { + const parsed = parsePaneKey(paneKey) + const leaf = parsed + ? this.leaves.get(this.getLeafKey(parsed.tabId, parsed.leafId)) + : undefined + addCurrent(paneKey, leaf?.ptyId ? this.ptysById.get(leaf.ptyId) : undefined) + } + for (const pty of this.ptysById.values()) { + if (pty.paneKey) { + addCurrent(pty.paneKey, pty) + } + } + const currentPtyByPaneKey = new Map() + for (const [paneKey, candidates] of currentCandidates) { + const connected = [...candidates].filter((pty) => pty.connected) + if (connected.length === 1) { + currentPtyByPaneKey.set(paneKey, connected[0]!) + } else if (connected.length === 0 && candidates.size === 1) { + currentPtyByPaneKey.set(paneKey, [...candidates][0]!) + } + } + + const persistedSurfaceByKey = new Map() + const indexedSessionHostIds = new Set() + for (const entry of entries) { + if (!entry.worktreeId) { + continue + } + const sessionHostId = this.tryGetWorkspaceSessionHostIdForWorktree(entry.worktreeId) + if (!sessionHostId || indexedSessionHostIds.has(sessionHostId)) { + continue + } + indexedSessionHostIds.add(sessionHostId) + const session = this.store?.getWorkspaceSession?.(sessionHostId) + for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) { + const hostKeys = this.getPersistedRestoredAgentHostKeys(sessionHostId, ptyId, inventory) + for (const hostKey of hostKeys) { + const key = restoredAgentSurfaceKey( + hostKey, + runtimeWorktreeIdentityKey(binding.worktreeId), + binding.paneKey + ) + const candidate = { ptyId, ...binding } + const existing = persistedSurfaceByKey.get(key) + if (existing === undefined) { + persistedSurfaceByKey.set(key, candidate) + } else if ( + existing !== null && + (existing.ptyId !== candidate.ptyId || + existing.incarnationId !== candidate.incarnationId) + ) { + persistedSurfaceByKey.set(key, null) + } + } + } + } + return { currentPtyByPaneKey, persistedSurfaceByKey, inventory } + } + + private getPersistedRestoredAgentHostKeys( + sessionHostId: ExecutionHostId, + ptyId: string, + inventory: PtyControllerInventory | null + ): readonly string[] { + const encodedHost = getPtyExecutionHost(ptyId) + if (encodedHost && encodedHost !== 'foreign') { + return [encodedHost] + } + const wslDistro = inventory?.terminalIdentityByPtyId.get(ptyId)?.wslDistro?.trim() + return wslDistro ? [sessionHostId, `wsl:${wslDistro.toLowerCase()}`] : [sessionHostId] + } + + private getRestoredAgentEvidenceIdentity(binding: RestoredAgentAuthorityBinding): string { + return makePtyProcessEvidenceIdentity( + binding.incarnationId, + this.runtimeId, + binding.lifecycleGeneration + ) + } + private getRestoredAgentAuthorityHook( entry: AgentStatusIpcPayload ): RestoredAgentAuthorityHook | null { @@ -20781,29 +20928,6 @@ export class OrcaRuntimeService { } } - private getExactPtyRecordForPaneKey(paneKey: string): RuntimePtyWorktreeRecord | null { - const parsed = parsePaneKey(paneKey) - const leaf = parsed ? this.leaves.get(this.getLeafKey(parsed.tabId, parsed.leafId)) : undefined - const candidates = new Set() - if (leaf?.ptyId) { - const pty = this.ptysById.get(leaf.ptyId) - if (pty) { - candidates.add(pty) - } - } - for (const pty of this.ptysById.values()) { - if (pty.paneKey === paneKey) { - candidates.add(pty) - } - } - const connected = [...candidates].filter((pty) => pty.connected) - return connected.length === 1 - ? connected[0]! - : candidates.size === 1 - ? [...candidates][0]! - : null - } - private ptyMatchesRestoredAgentHost(pty: RuntimePtyWorktreeRecord, hostKey: string): boolean { const encodedHost = getPtyExecutionHost(pty.ptyId) if (encodedHost === 'foreign') { @@ -39025,6 +39149,8 @@ export function resolveWorktreeScanCacheTtlMs(repo: Pick { diff --git a/src/main/runtime/pty-process-inspection.ts b/src/main/runtime/pty-process-inspection.ts new file mode 100644 index 00000000000..75aae773bd6 --- /dev/null +++ b/src/main/runtime/pty-process-inspection.ts @@ -0,0 +1,60 @@ +export type PtyProcessInspectionSource = { + getForegroundProcess(ptyId: string): Promise + inspectProcess?: (ptyId: string) => Promise<{ + foregroundProcess: string | null + hasChildProcesses: boolean + unavailable?: true + }> + hasChildProcesses?: (ptyId: string) => Promise +} + +export type PtyProcessLivenessEvidence = + | { + status: 'live' + foregroundProcess: string | null + hasChildProcesses: boolean | null + } + | { status: 'unverifiable'; reason: string } + | { status: 'exited' } + +export async function inspectPtyProcess( + source: PtyProcessInspectionSource, + ptyId: string +): Promise { + try { + if (source.inspectProcess) { + const inspection = await source.inspectProcess(ptyId) + return inspection.unavailable + ? { status: 'unverifiable', reason: 'process inspection unavailable' } + : { + status: 'live', + foregroundProcess: inspection.foregroundProcess, + hasChildProcesses: inspection.hasChildProcesses + } + } + const foregroundProcess = await source.getForegroundProcess(ptyId) + return { + status: 'live', + foregroundProcess, + hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null + } + } catch (error) { + if (isTerminalGoneError(error)) { + return { status: 'exited' } + } + throw error + } +} + +export function describeProcessInspectionError(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function isTerminalGoneError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error) + const code = + error && typeof error === 'object' && 'code' in error + ? String((error as { code?: unknown }).code) + : '' + return message === 'terminal_gone' || code === 'terminal_gone' +} diff --git a/src/main/runtime/pty-process-liveness-broker.test.ts b/src/main/runtime/pty-process-liveness-broker.test.ts index 2b81dba9ae6..7c875209d7d 100644 --- a/src/main/runtime/pty-process-liveness-broker.test.ts +++ b/src/main/runtime/pty-process-liveness-broker.test.ts @@ -48,6 +48,80 @@ describe('PtyProcessLivenessBroker', () => { expect(inspectProcess).toHaveBeenCalledOnce() }) + it('bounds hanging probes across PTYs and prunes retired consumer entries', async () => { + vi.useFakeTimers() + const resolvers: ((value: { + foregroundProcess: string + hasChildProcesses: boolean + }) => void)[] = [] + const inspectProcess = vi.fn( + () => + new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => { + resolvers.push(resolve) + }) + ) + const source = { getForegroundProcess: vi.fn(async () => null), inspectProcess } + const broker = new PtyProcessLivenessBroker({ timeoutMs: 100, maxConcurrentProbes: 2 }) + + const reads = Array.from({ length: 5 }, (_, index) => + broker.inspect({ + source, + ptyId: `pty-${index}`, + identity: `incarnation-${index}`, + consumerId: 'restored-agent' + }) + ) + await vi.advanceTimersByTimeAsync(100) + await Promise.all(reads) + + expect(inspectProcess).toHaveBeenCalledTimes(2) + expect(broker.getActiveProbeCount()).toBe(2) + expect(broker.getEntryCount()).toBe(2) + + broker.retainConsumerEvidence('restored-agent', []) + expect(broker.getEntryCount()).toBe(0) + expect(broker.getActiveProbeCount()).toBe(2) + + for (const resolve of resolvers) { + resolve({ foregroundProcess: 'codex', hasChildProcesses: true }) + } + await vi.runAllTimersAsync() + expect(broker.getActiveProbeCount()).toBe(0) + }) + + it('does not prune evidence shared with an unscoped broker consumer', async () => { + let resolveInspection!: (value: { + foregroundProcess: string + hasChildProcesses: boolean + }) => void + const inspection = new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>( + (resolve) => { + resolveInspection = resolve + } + ) + const source = { + getForegroundProcess: vi.fn(async () => null), + inspectProcess: vi.fn(() => inspection) + } + const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 }) + + const restored = broker.inspect({ + source, + ptyId: 'pty-1', + identity: 'incarnation-a', + consumerId: 'restored-agent' + }) + const foreground = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' }) + broker.retainConsumerEvidence('restored-agent', []) + + expect(broker.getEntryCount()).toBe(1) + resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true }) + await expect(Promise.all([restored, foreground])).resolves.toEqual([ + expect.objectContaining({ status: 'live' }), + expect.objectContaining({ status: 'live' }) + ]) + }) + it('caches live evidence across sequential clients', async () => { const inspectProcess = vi.fn(async () => ({ foregroundProcess: 'codex', diff --git a/src/main/runtime/pty-process-liveness-broker.ts b/src/main/runtime/pty-process-liveness-broker.ts index 75a5a487c9d..01bd0a1ed3a 100644 --- a/src/main/runtime/pty-process-liveness-broker.ts +++ b/src/main/runtime/pty-process-liveness-broker.ts @@ -1,25 +1,20 @@ -export type PtyProcessInspectionSource = { - getForegroundProcess(ptyId: string): Promise - inspectProcess?: (ptyId: string) => Promise<{ - foregroundProcess: string | null - hasChildProcesses: boolean - unavailable?: true - }> - hasChildProcesses?: (ptyId: string) => Promise -} +import { + describeProcessInspectionError, + inspectPtyProcess, + type PtyProcessInspectionSource, + type PtyProcessLivenessEvidence +} from './pty-process-inspection' -export type PtyProcessLivenessEvidence = - | { - status: 'live' - foregroundProcess: string | null - hasChildProcesses: boolean | null - } - | { status: 'unverifiable'; reason: string } - | { status: 'exited' } +export type { + PtyProcessInspectionSource, + PtyProcessLivenessEvidence +} from './pty-process-inspection' type PtyProcessEvidenceEntry = { source: PtyProcessInspectionSource identity: string + consumerIds: Set + hasUnscopedConsumer: boolean freshness: number owningInventoryObservedPty: boolean failureCount: number @@ -31,6 +26,7 @@ type PtyProcessEvidenceEntry = { export type PtyProcessLivenessBrokerOptions = { timeoutMs: number + maxConcurrentProbes?: number liveTtlMs?: number unavailableBackoffBaseMs?: number unavailableBackoffMaxMs?: number @@ -41,10 +37,12 @@ export type PtyProcessLivenessBrokerOptions = { const DEFAULT_LIVE_TTL_MS = 10_000 const DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS = 3_000 const DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS = 30_000 +const DEFAULT_MAX_CONCURRENT_PROBES = 8 export class PtyProcessLivenessBroker { private readonly entries = new Map() private readonly now: () => number + private activeProbeCount = 0 constructor(private readonly options: PtyProcessLivenessBrokerOptions) { this.now = options.now ?? Date.now @@ -59,6 +57,7 @@ export class PtyProcessLivenessBroker { deadline?: number waitForSettlement?: boolean owningInventoryObservedPty?: boolean + consumerId?: string }): Promise { const freshness = args.freshness ?? 0 const waitMs = args.waitForSettlement @@ -71,6 +70,9 @@ export class PtyProcessLivenessBroker { ) ) const existing = this.entries.get(args.ptyId) + if (existing?.source === args.source && existing.identity === args.identity) { + this.rememberConsumer(existing, args.consumerId) + } if ( existing?.source === args.source && existing.identity === args.identity && @@ -121,6 +123,12 @@ export class PtyProcessLivenessBroker { if (waitMs === 0) { return Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' }) } + if (this.activeProbeCount >= this.maxConcurrentProbes()) { + return Promise.resolve({ + status: 'unverifiable', + reason: 'process inspection capacity unavailable' + }) + } const failureCount = existing?.source === args.source && existing.identity === args.identity @@ -129,6 +137,8 @@ export class PtyProcessLivenessBroker { const entry: PtyProcessEvidenceEntry = { source: args.source, identity: args.identity, + consumerIds: new Set(args.consumerId ? [args.consumerId] : []), + hasUnscopedConsumer: args.consumerId === undefined, freshness, owningInventoryObservedPty: args.owningInventoryObservedPty === true, failureCount, @@ -137,7 +147,8 @@ export class PtyProcessLivenessBroker { timedOut: false, probe: null } - const probe = this.runProbe(args.source, args.ptyId) + this.activeProbeCount += 1 + const probe = inspectPtyProcess(args.source, args.ptyId) .then((evidence) => { const reconciled = evidence.status === 'exited' && entry.owningInventoryObservedPty @@ -169,7 +180,10 @@ export class PtyProcessLivenessBroker { entry.probe = null entry.timedOut = false entry.failureCount += 1 - entry.evidence = { status: 'unverifiable', reason: describeError(error) } + entry.evidence = { + status: 'unverifiable', + reason: describeProcessInspectionError(error) + } entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount) } try { @@ -177,9 +191,11 @@ export class PtyProcessLivenessBroker { } catch { // Diagnostic observers cannot change the authority verdict. } - return { status: 'unverifiable', reason: describeError(error) } + return { status: 'unverifiable', reason: describeProcessInspectionError(error) } }) - entry.probe = probe + entry.probe = probe.finally(() => { + this.activeProbeCount -= 1 + }) this.entries.set(args.ptyId, entry) return this.waitForProbe(args.ptyId, entry, waitMs) } @@ -192,6 +208,27 @@ export class PtyProcessLivenessBroker { this.entries.clear() } + retainConsumerEvidence( + consumerId: string, + retained: readonly Readonly<{ ptyId: string; identity: string }>[] + ): void { + const retainedKeys = new Set( + retained.map(({ ptyId, identity }) => processEvidenceKey(ptyId, identity)) + ) + for (const [ptyId, entry] of this.entries) { + if ( + !entry.consumerIds.has(consumerId) || + retainedKeys.has(processEvidenceKey(ptyId, entry.identity)) + ) { + continue + } + entry.consumerIds.delete(consumerId) + if (!entry.hasUnscopedConsumer && entry.consumerIds.size === 0) { + this.entries.delete(ptyId) + } + } + } + getPendingCount(): number { let count = 0 for (const entry of this.entries.values()) { @@ -202,33 +239,12 @@ export class PtyProcessLivenessBroker { return count } - private async runProbe( - source: PtyProcessInspectionSource, - ptyId: string - ): Promise { - try { - if (source.inspectProcess) { - const inspection = await source.inspectProcess(ptyId) - return inspection.unavailable - ? { status: 'unverifiable', reason: 'process inspection unavailable' } - : { - status: 'live', - foregroundProcess: inspection.foregroundProcess, - hasChildProcesses: inspection.hasChildProcesses - } - } - const foregroundProcess = await source.getForegroundProcess(ptyId) - return { - status: 'live', - foregroundProcess, - hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null - } - } catch (error) { - if (isTerminalGoneError(error)) { - return { status: 'exited' } - } - throw error - } + getActiveProbeCount(): number { + return this.activeProbeCount + } + + getEntryCount(): number { + return this.entries.size } private waitForProbe( @@ -265,6 +281,21 @@ export class PtyProcessLivenessBroker { return Math.min(max, base * 2 ** Math.max(0, failureCount - 1)) } + private maxConcurrentProbes(): number { + const configured = this.options.maxConcurrentProbes ?? DEFAULT_MAX_CONCURRENT_PROBES + return Number.isFinite(configured) && configured > 0 + ? Math.max(1, Math.floor(configured)) + : DEFAULT_MAX_CONCURRENT_PROBES + } + + private rememberConsumer(entry: PtyProcessEvidenceEntry, consumerId: string | undefined): void { + if (consumerId === undefined) { + entry.hasUnscopedConsumer = true + } else { + entry.consumerIds.add(consumerId) + } + } + private storeUnverifiable(entry: PtyProcessEvidenceEntry, reason: string): void { entry.failureCount += 1 entry.evidence = { status: 'unverifiable', reason } @@ -272,15 +303,6 @@ export class PtyProcessLivenessBroker { } } -function isTerminalGoneError(error: unknown): boolean { - const message = error instanceof Error ? error.message : String(error) - const code = - error && typeof error === 'object' && 'code' in error - ? String((error as { code?: unknown }).code) - : '' - return message === 'terminal_gone' || code === 'terminal_gone' -} - -function describeError(error: unknown): string { - return error instanceof Error ? error.message : String(error) +function processEvidenceKey(ptyId: string, identity: string): string { + return JSON.stringify([ptyId, identity]) } diff --git a/src/main/runtime/restored-agent-authority-resolver.test.ts b/src/main/runtime/restored-agent-authority-resolver.test.ts index e7125494ae0..78343a9e348 100644 --- a/src/main/runtime/restored-agent-authority-resolver.test.ts +++ b/src/main/runtime/restored-agent-authority-resolver.test.ts @@ -42,9 +42,13 @@ describe('RestoredAgentAuthorityResolver', () => { it('keeps one hook fenced to its first process incarnation', () => { const resolver = new RestoredAgentAuthorityResolver() - expect(resolver.resolve({ hook, current: binding(), persisted: null }).binding).toEqual( - binding() - ) + expect( + resolver.resolve({ + hook, + current: binding(), + persisted: binding({ source: 'persisted' }) + }).binding + ).toEqual(binding()) expect( resolver.resolve({ @@ -57,7 +61,11 @@ describe('RestoredAgentAuthorityResolver', () => { it('treats lifecycle generations as fallback identity when incarnation exists', () => { const resolver = new RestoredAgentAuthorityResolver() - resolver.resolve({ hook, current: binding(), persisted: null }) + resolver.resolve({ + hook, + current: binding(), + persisted: binding({ source: 'persisted' }) + }) expect( resolver.resolve({ @@ -68,6 +76,91 @@ describe('RestoredAgentAuthorityResolver', () => { ).toEqual(binding({ lifecycleGeneration: 2 })) }) + it('does not confuse a generation-shaped incarnation with fallback identity', () => { + const resolver = new RestoredAgentAuthorityResolver() + resolver.resolve({ + hook, + current: binding({ incarnationId: null, lifecycleGeneration: 1 }), + persisted: binding({ + incarnationId: null, + lifecycleGeneration: 1, + source: 'persisted' + }) + }) + + expect( + resolver.resolve({ + hook, + current: binding({ incarnationId: 'generation:1', lifecycleGeneration: 99 }), + persisted: null + }) + ).toEqual({ binding: null, hasExactBinding: false }) + }) + + it('keeps process authority stable when a terminal handle is learned', () => { + const resolver = new RestoredAgentAuthorityResolver() + resolver.resolve({ + hook, + current: binding(), + persisted: binding({ source: 'persisted' }) + }) + + expect( + resolver.resolve({ + hook, + current: binding({ terminalHandle: 'term_learned' }), + persisted: null + }).binding + ).toEqual(binding({ terminalHandle: 'term_learned' })) + }) + + it('uses a hook terminal handle as a constraint when one is present', () => { + const resolver = new RestoredAgentAuthorityResolver() + const handleHook = { ...hook, terminalHandle: 'term_expected' } + + expect( + resolver.resolve({ + hook: handleHook, + current: binding({ terminalHandle: 'term_other' }), + persisted: null + }) + ).toEqual({ binding: null, hasExactBinding: false }) + }) + + it('requires persisted authority before accepting a current binding', () => { + const resolver = new RestoredAgentAuthorityResolver() + + expect(resolver.resolve({ hook, current: binding(), persisted: null })).toEqual({ + binding: null, + hasExactBinding: false + }) + }) + + it('seeds from persisted authority and later accepts matching current authority', () => { + const resolver = new RestoredAgentAuthorityResolver() + const persisted = binding({ source: 'persisted' }) + + expect(resolver.resolve({ hook, current: null, persisted })).toEqual({ + binding: persisted, + hasExactBinding: true + }) + expect(resolver.resolve({ hook, current: binding(), persisted: null }).binding).toEqual( + binding() + ) + }) + + it('does not let a replacement current binding borrow first authority', () => { + const resolver = new RestoredAgentAuthorityResolver() + + expect( + resolver.resolve({ + hook, + current: binding({ ptyId: 'pty-replacement', incarnationId: 'incarnation-b' }), + persisted: binding({ source: 'persisted' }) + }) + ).toEqual({ binding: null, hasExactBinding: false }) + }) + it('rejects disagreement between current and persisted bindings', () => { const resolver = new RestoredAgentAuthorityResolver() @@ -82,15 +175,20 @@ describe('RestoredAgentAuthorityResolver', () => { it('forgets commitments after their restored hook disappears', () => { const resolver = new RestoredAgentAuthorityResolver() - resolver.resolve({ hook, current: binding(), persisted: null }) + resolver.resolve({ + hook, + current: binding(), + persisted: binding({ source: 'persisted' }) + }) resolver.retain(new Set()) + const replacement = binding({ incarnationId: 'incarnation-b', source: 'persisted' }) expect( resolver.resolve({ hook, - current: binding({ incarnationId: 'incarnation-b' }), - persisted: null + current: null, + persisted: replacement }).binding - ).toEqual(binding({ incarnationId: 'incarnation-b' })) + ).toEqual(replacement) }) }) diff --git a/src/main/runtime/restored-agent-authority-resolver.ts b/src/main/runtime/restored-agent-authority-resolver.ts index 14e4414680d..3a397e865ca 100644 --- a/src/main/runtime/restored-agent-authority-resolver.ts +++ b/src/main/runtime/restored-agent-authority-resolver.ts @@ -35,12 +35,15 @@ export class RestoredAgentAuthorityResolver { if (current && persisted && !sameRestoredAgentProcessAuthority(current, persisted)) { return { binding: null, hasExactBinding: false } } + const existing = this.commitmentByHookIdentity.get(args.hook.identity) + if (!existing && !persisted) { + return { binding: null, hasExactBinding: false } + } const binding = current ?? persisted if (!binding) { return { binding: null, hasExactBinding: false } } const commitment = processAuthorityKey(binding) - const existing = this.commitmentByHookIdentity.get(args.hook.identity) if (existing && existing !== commitment) { return { binding: null, hasExactBinding: false } } @@ -83,7 +86,8 @@ function processAuthorityKey(binding: RestoredAgentAuthorityBinding): string { binding.worktreeKey, binding.paneKey, binding.ptyId, - binding.incarnationId ?? `generation:${binding.lifecycleGeneration}`, - binding.terminalHandle ?? null + binding.incarnationId === null + ? ['generation', binding.lifecycleGeneration] + : ['incarnation', binding.incarnationId] ]) }