fix(runtime): fence restored agent liveness evidence

This commit is contained in:
Jinwoo-H
2026-08-23 21:45:27 -04:00
parent ba98df093b
commit 77b364984e
5 changed files with 922 additions and 163 deletions
+9 -5
View File
@@ -10293,7 +10293,7 @@
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
"cd mobile && pnpm exec vitest run src/worktree/agent-row-display.test.ts src/transport/stable-logical-rpc-client.test.ts",
"pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1",
"pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1",
@@ -10305,6 +10305,7 @@
"src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts",
"src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts",
"src/main/runtime/orca-runtime.test.ts",
"src/main/runtime/pty-process-liveness-broker.test.ts",
"src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts",
"src/cli/handlers/orchestration.test.ts",
"src/cli/handlers/orchestration-lifecycle-rejection.test.ts",
@@ -10356,7 +10357,10 @@
"retained renderer output remains readable through the recovered terminal",
"capable worktree inventory projects restored live and unverifiable identities but omits exited shell-only agents",
"a connected live PTY remains inventoried when its user-visible tab is missing",
"legacy clients retain the exact restored-row suppression without a process inspection"
"legacy clients retain the exact restored-row suppression",
"inspection timeout, transport loss, missing bindings, and stale incarnations remain unverifiable",
"terminal_gone is the only inspection rejection that retires the exact restored row",
"sequential inventory clients reuse bounded process evidence instead of polling each restored row"
]
},
{
@@ -10504,10 +10508,10 @@
"date": "2026-08-23",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
"result": "passed",
"durationSeconds": 15.33,
"summary": "Seven files passed 1,264 tests with one skip, covering live, unverifiable, exited, hidden-tab, legacy and request-level capability projection plus every encrypted desktop/CLI client capability advertisement. Mobile typecheck and 25 focused mobile tests separately passed."
"summary": "Eight focused runtime and wire files passed 1,273 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, cold persisted binding, incarnation and hook replacement races, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, and encrypted desktop/CLI capability advertisement. Mobile typecheck and 25 focused mobile tests passed separately."
},
{
"date": "2026-08-13",
@@ -10686,7 +10690,7 @@
},
"performanceBudget": {
"required": true,
"evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. Capable worktree.ps requests inspect only restored-unconfirmed candidates, deduplicate concurrent and same-request reads by PTY, and do no inspection for legacy clients, confirmed rows, unrelated PTYs, or restored rows without a retained PTY. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path."
"evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, a hung provider retains at most one unresolved probe per identity, and every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path."
},
"promotionCriteria": [
"Collect 100 consecutive passes or 14 days of stable CI history on macOS, Linux, and Windows.",
+299 -7
View File
@@ -22,7 +22,10 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state
import type { WorkspaceLineage, WorktreeLineage } from '../../shared/worktree/lineage-types'
import type { WorktreeMeta } from '../../shared/worktree/meta-types'
import type { Worktree } from '../../shared/worktree/types'
import { AGENT_STATUS_STALE_AFTER_MS } from '../../shared/agent-status-types'
import {
AGENT_STATUS_STALE_AFTER_MS,
type AgentStatusIpcPayload
} from '../../shared/agent-status-types'
import {
reviewHeadRemoteRefComponent,
REVIEW_HEAD_FETCH_TIMEOUT_MS
@@ -9873,12 +9876,12 @@ describe('OrcaRuntimeService', () => {
expect(
(
runtime as unknown as {
ptyForegroundProcessReads: Map<string, unknown>
ptyProcessLiveness: { getPendingCount(): number }
}
).ptyForegroundProcessReads.size
).ptyProcessLiveness.getPendingCount()
).toBe(0)
)
await Promise.resolve()
await new Promise<void>((resolve) => setImmediate(resolve))
getForegroundProcess.mockResolvedValueOnce('zsh')
runtime.onPtyData('pty-1', '\x1b]0;other cwd\x07', 102)
@@ -37744,12 +37747,146 @@ describe('OrcaRuntimeService', () => {
leaves: []
})
const { worktrees } = await runtime.getWorktreePs()
const { worktrees } = await runtime.getWorktreePs(undefined, true, false)
const summary = worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)
expect(summary).toMatchObject({ hasHostSidebarActivity: false, status: 'inactive', agents: [] })
})
it('keeps a restored row unverifiable when current PTY bookkeeping is missing', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const now = Date.now()
const runtime = new OrcaRuntimeService(store, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'tab-1',
state: 'working',
prompt: 'await provider adoption',
agentType: 'codex',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 100,
restoredUnconfirmed: true
}
]
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
tabs: [
{
tabId: 'tab-1',
worktreeId: TEST_WORKTREE_ID,
title: 'Codex',
activeLeafId: leafId,
layout: null
}
],
leaves: []
})
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
expect.objectContaining({
paneKey,
restoredUnconfirmed: true,
agentLiveness: 'unverifiable'
})
])
})
it('uses a cold persisted binding without certifying a replacement incarnation live', async () => {
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
const session = makeWorkspaceSessionWithHeadlessTerminal({
terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId }
})
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
const now = Date.now()
const inspectProcess = vi.fn(async () => ({
foregroundProcess: 'codex',
hasChildProcesses: true
}))
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'host-tab',
state: 'working',
prompt: 'cold persisted agent',
agentType: 'codex',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 100,
restoredUnconfirmed: true
}
]
})
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => 'codex'),
inspectProcess,
listProcesses: vi.fn(async () => [])
})
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
expect.objectContaining({
paneKey,
restoredUnconfirmed: true,
agentLiveness: 'unverifiable'
})
])
expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('persisted-pty')
})
it('omits a cold restored row only after terminal_gone confirms exact absence', async () => {
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
const session = makeWorkspaceSessionWithHeadlessTerminal({
terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId }
})
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
const now = Date.now()
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'host-tab',
state: 'working',
prompt: 'completed while runtime was down',
agentType: 'codex',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 100,
restoredUnconfirmed: true
}
]
})
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => null),
inspectProcess: vi.fn(async () => {
throw new Error('terminal_gone')
}),
listProcesses: vi.fn(async () => [])
})
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
[]
)
})
it('projects a restored agent after the execution host confirms its exact PTY is live', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
@@ -37833,7 +37970,7 @@ describe('OrcaRuntimeService', () => {
agentLiveness: 'live'
})
])
expect(inspectProcess).toHaveBeenCalledTimes(2)
expect(inspectProcess).toHaveBeenCalledOnce()
})
it('keeps a restored agent visible as unverifiable when PTY inspection loses contact', async () => {
@@ -37900,6 +38037,161 @@ describe('OrcaRuntimeService', () => {
])
})
it('fences a restored-agent inspection across PTY reincarnation', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const now = Date.now()
let resolveInspection!: (value: {
foregroundProcess: string
hasChildProcesses: boolean
}) => void
const inspectProcess = vi.fn(
() =>
new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => {
resolveInspection = resolve
})
)
const runtime = new OrcaRuntimeService(store, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'tab-1',
state: 'working',
prompt: 'old incarnation',
agentType: 'codex',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 100,
restoredUnconfirmed: true
}
]
})
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => 'codex'),
inspectProcess,
listProcesses: vi.fn(async () => [])
})
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
tabId: 'tab-1',
leafId,
incarnationId: 'incarnation-a'
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
tabs: [
{
tabId: 'tab-1',
worktreeId: TEST_WORKTREE_ID,
title: 'Codex',
activeLeafId: leafId,
layout: null
}
],
leaves: [
{
tabId: 'tab-1',
worktreeId: TEST_WORKTREE_ID,
leafId,
paneRuntimeId: 1,
ptyId: 'pty-1'
}
]
})
const inventory = runtime.getWorktreePs(undefined, true, true)
await vi.waitFor(() => expect(inspectProcess).toHaveBeenCalledOnce())
runtime.onPtySpawned('pty-1', 'incarnation-b', { awaitsRegistration: false })
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
const { worktrees } = await inventory
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
expect.objectContaining({
paneKey,
restoredUnconfirmed: true,
agentLiveness: 'unverifiable'
})
])
})
it('does not publish a restored hook snapshot replaced during inspection', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
const now = Date.now()
let rows: AgentStatusIpcPayload[] = [
{
paneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'tab-1',
state: 'working',
prompt: 'stale hook',
agentType: 'codex',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 100,
restoredUnconfirmed: true
}
]
let resolveInspection!: (value: {
foregroundProcess: string
hasChildProcesses: boolean
}) => void
const inspectProcess = vi.fn(
() =>
new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => {
resolveInspection = resolve
})
)
const runtime = new OrcaRuntimeService(store, undefined, {
getAgentStatusSnapshot: () => rows
})
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: vi.fn(async () => 'codex'),
inspectProcess,
listProcesses: vi.fn(async () => [])
})
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
tabId: 'tab-1',
leafId,
incarnationId: 'incarnation-a'
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
tabs: [
{
tabId: 'tab-1',
worktreeId: TEST_WORKTREE_ID,
title: 'Codex',
activeLeafId: leafId,
layout: null
}
],
leaves: [
{
tabId: 'tab-1',
worktreeId: TEST_WORKTREE_ID,
leafId,
paneRuntimeId: 1,
ptyId: 'pty-1'
}
]
})
const inventory = runtime.getWorktreePs(undefined, true, true)
await vi.waitFor(() => expect(inspectProcess).toHaveBeenCalledOnce())
rows = []
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
const { worktrees } = await inventory
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
[]
)
})
it('omits an exited restored agent after the exact PTY is confirmed shell-only', async () => {
const leafId = '33333333-3333-4333-8333-333333333333'
const paneKey = `tab-1:${leafId}`
@@ -38018,7 +38310,7 @@ describe('OrcaRuntimeService', () => {
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
[]
)
expect(inspectProcess).not.toHaveBeenCalled()
expect(inspectProcess).toHaveBeenCalledOnce()
})
it('uses mirrored tab ownership after a workspace rename instead of stale hook attribution', async () => {
+230 -151
View File
@@ -660,6 +660,7 @@ import type { RuntimeBrowserCommands } from './orca-runtime-browser'
import { createRuntimeBrowserCommands } from './runtime-browser-commands-factory'
import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency'
import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity'
import { PtyProcessLivenessBroker } from './pty-process-liveness-broker'
import {
buildHeadlessTerminalSplitLayout,
countTerminalLayoutLeaves,
@@ -1631,26 +1632,21 @@ type PtyForegroundProcessRead = {
available: boolean
}
type PtyForegroundProcessReadEntry = {
controller: RuntimePtyController
startedAfterTitleObservation: number
promise: Promise<PtyForegroundProcessRead>
}
type RestoredAgentPtyInspection = {
controller: RuntimePtyController
foregroundProcess: string | null
hasChildProcesses: boolean
unavailable: boolean
}
type RestoredAgentPtyInspectionEntry = {
controller: RuntimePtyController
promise: Promise<RestoredAgentPtyInspection>
}
type RestoredAgentLiveness = 'live' | 'unverifiable' | 'exited'
type RestoredAgentLivenessResolution = {
liveness: RestoredAgentLiveness
hookIdentity: string
hasExactBinding: boolean
}
type RestoredAgentPtyBinding = {
ptyId: string
incarnationId: PtyIncarnationId | null
lifecycleGeneration: number
source: 'current' | 'persisted'
}
function copySleepingAgentLaunchConfig(
config: SleepingAgentLaunchConfig
): SleepingAgentLaunchConfig {
@@ -3184,8 +3180,12 @@ export class OrcaRuntimeService {
private worktreeAdminFingerprintProbes = new Set<string>()
private cloneInFlightByPath = new Map<string, Promise<void>>()
private ptyForegroundAgentRefreshes = new Map<string, PtyForegroundAgentRefresh>()
private ptyForegroundProcessReads = new Map<string, PtyForegroundProcessReadEntry>()
private restoredAgentPtyInspections = new Map<string, RestoredAgentPtyInspectionEntry>()
private readonly ptyProcessLiveness = new PtyProcessLivenessBroker({
timeoutMs: PTY_PROCESS_INSPECTION_TIMEOUT_MS,
onInspectionError: (ptyId, error) => {
console.warn(`[runtime] PTY process inspection failed for ${ptyId}`, error)
}
})
private ptyDelayedForegroundSnapshotTitleObservations = new Map<string, number>()
// Why a set and not a timer: the intent is retired by the exit it explains, or
// by the next lifecycle generation on that id (advancePtyLifecycleGeneration),
@@ -6181,7 +6181,10 @@ export class OrcaRuntimeService {
// Why: CLI terminal writes must go through the main-owned PTY registry
// instead of tunneling back through renderer IPC, or live handles could
// drift from the process they are supposed to control during reloads.
this.ptyController = controller
if (this.ptyController !== controller) {
this.ptyProcessLiveness.invalidateAll()
this.ptyController = controller
}
}
setNotifier(notifier: RuntimeNotifier | null): void {
@@ -10795,6 +10798,12 @@ export class OrcaRuntimeService {
incarnationId?: PtyIncarnationId,
options: { awaitsRegistration?: boolean } = {}
): void {
const previousIncarnation = this.ptysById.get(ptyId)?.incarnationId
if (previousIncarnation && incarnationId && previousIncarnation !== incarnationId) {
this.advancePtyLifecycleGeneration(ptyId)
} else {
this.ptyProcessLiveness.invalidate(ptyId)
}
this.forgetPtyLivenessVerdict(ptyId)
if (options.awaitsRegistration !== false) {
// Why: surface absence cannot distinguish an in-flight admission from a completed headless lifecycle.
@@ -12262,6 +12271,7 @@ export class OrcaRuntimeService {
private advancePtyLifecycleGeneration(ptyId: string): void {
this.ptyLifecycleGenerationById.set(ptyId, this.nextPtyLifecycleGeneration++)
this.ptyProcessLiveness.invalidate(ptyId)
// Why: a stop whose exit never arrived would otherwise stay armed across a
// same-id respawn and label the NEXT process's crash an operator close —
// the exact lie this cause model exists to remove.
@@ -19050,73 +19060,44 @@ export class OrcaRuntimeService {
private readPtyForegroundProcessFromController(
ptyId: string,
afterTitleObservation = 0
afterTitleObservation = 0,
options: { reuseSettled?: boolean } = {}
): Promise<PtyForegroundProcessRead> | null {
const controller = this.ptyController
if (!controller) {
return null
}
const pending = this.ptyForegroundProcessReads.get(ptyId)
if (
pending?.controller === controller &&
pending.startedAfterTitleObservation >= afterTitleObservation
) {
return pending.promise
}
if (pending?.controller === controller) {
return pending.promise.then(
() =>
this.readPtyForegroundProcessFromController(ptyId, afterTitleObservation) ?? {
controller,
process: null,
available: false
}
)
}
const unavailable: PtyForegroundProcessRead = {
controller,
process: null,
available: false
}
let processRead: Promise<string | null>
try {
processRead = Promise.resolve(controller.getForegroundProcess(ptyId))
} catch {
const entry: PtyForegroundProcessReadEntry = {
const identity = this.getPtyProcessEvidenceIdentity(ptyId)
return this.ptyProcessLiveness
.inspect({
source: controller,
ptyId,
identity,
freshness: afterTitleObservation,
reuseSettled: options.reuseSettled
})
.then((evidence) => ({
controller,
startedAfterTitleObservation: afterTitleObservation,
promise: Promise.resolve(unavailable)
}
entry.promise = entry.promise.finally(() => {
if (this.ptyForegroundProcessReads.get(ptyId) === entry) {
this.ptyForegroundProcessReads.delete(ptyId)
}
})
this.ptyForegroundProcessReads.set(ptyId, entry)
return entry.promise
}
let entry: PtyForegroundProcessReadEntry
const promise = processRead
.then((process) => ({ controller, process, available: true }))
.catch(() => unavailable)
.finally(() => {
if (this.ptyForegroundProcessReads.get(ptyId) === entry) {
this.ptyForegroundProcessReads.delete(ptyId)
}
})
entry = {
controller,
startedAfterTitleObservation: afterTitleObservation,
promise
}
this.ptyForegroundProcessReads.set(ptyId, entry)
return entry.promise
process: evidence.status === 'live' ? evidence.foregroundProcess : null,
available: evidence.status === 'live'
}))
}
private getPtyProcessEvidenceIdentity(ptyId: string): string {
return (
this.ptysById.get(ptyId)?.incarnationId ??
`runtime:${this.runtimeId}:${this.getPtyLifecycleGeneration(ptyId)}`
)
}
private confirmPtyAgentExit(ptyId: string): void {
const pty = this.ptysById.get(ptyId)
const titleObservedAt = pty?.lastOscTitleAt ?? null
const foregroundRead = this.readPtyForegroundProcessFromController(ptyId, titleObservedAt ?? 0)
const foregroundRead = this.readPtyForegroundProcessFromController(
ptyId,
titleObservedAt ?? 0,
{ reuseSettled: false }
)
if (!pty?.connected || !foregroundRead) {
this.recordTerminalSideEffectFact(ptyId, { kind: 'agent-exited' })
return
@@ -19996,6 +19977,7 @@ export class OrcaRuntimeService {
visibilitySettings
)
)
const ptyProcessEvidenceDeadline = Date.now() + PTY_CONTROLLER_LIST_TIMEOUT_MS
// Why: worktree.ps backs the mobile sidebar, so it must use the same
// host-owned imported-worktree visibility gate as worktree.list/desktop.
const freshPtyLiveness = await this.refreshPtyWorktreeRecordsFromController(resolvedWorktrees)
@@ -20355,7 +20337,8 @@ export class OrcaRuntimeService {
missingRuntimeWorktreeIds,
mirroredWorktreeIdByTabId,
connectedPtyEvidence,
restoredAgentPresenceSupported
restoredAgentPresenceSupported,
ptyProcessEvidenceDeadline
)
const sorted = [...summaries.values()].sort(compareWorktreePs)
@@ -20380,7 +20363,8 @@ export class OrcaRuntimeService {
paneKeys: ReadonlySet<string>
ptyIds: ReadonlySet<string>
},
restoredAgentPresenceSupported: boolean
restoredAgentPresenceSupported: boolean,
ptyProcessEvidenceDeadline: number
): Promise<void> {
// Why: most agents report via hooks (agent-hooks/server), not OSC, so the
// hook snapshot is the primary source — same one the desktop sidebar reads.
@@ -20404,6 +20388,7 @@ export class OrcaRuntimeService {
updatedAt: number
restoredUnconfirmed?: boolean
agentLiveness?: Exclude<RestoredAgentLiveness, 'exited'>
restoredExactBinding?: boolean
}
>()
for (const snapshot of this.latestAgentStatusByPaneKey.values()) {
@@ -20425,13 +20410,20 @@ export class OrcaRuntimeService {
updatedAt: snapshot.updatedAt
})
}
const hookEntries = this.getAgentStatusSnapshotFn?.() ?? []
const hookEntriesAtInspection = this.getAgentStatusSnapshotFn?.() ?? []
const restoredAgentLiveness = restoredAgentPresenceSupported
? await this.resolveRestoredAgentLiveness(hookEntries)
: new Map<string, RestoredAgentLiveness>()
? await this.resolveRestoredAgentLiveness(hookEntriesAtInspection, ptyProcessEvidenceDeadline)
: new Map<string, RestoredAgentLivenessResolution>()
const hookEntries = restoredAgentPresenceSupported
? (this.getAgentStatusSnapshotFn?.() ?? hookEntriesAtInspection)
: hookEntriesAtInspection
for (const entry of hookEntries) {
const liveness =
const livenessResolution =
entry.restoredUnconfirmed === true ? restoredAgentLiveness.get(entry.paneKey) : undefined
const liveness =
livenessResolution?.hookIdentity === restoredAgentHookIdentity(entry)
? livenessResolution.liveness
: undefined
// Why: legacy clients ignore restored provenance, while a confirmed shell proves the old agent exited.
if (
entry.restoredUnconfirmed === true &&
@@ -20465,7 +20457,8 @@ export class OrcaRuntimeService {
...(entry.restoredUnconfirmed === true
? {
restoredUnconfirmed: true,
agentLiveness: liveness as Exclude<RestoredAgentLiveness, 'exited'>
agentLiveness: liveness as Exclude<RestoredAgentLiveness, 'exited'>,
restoredExactBinding: livenessResolution?.hasExactBinding === true
}
: {})
})
@@ -20501,7 +20494,9 @@ export class OrcaRuntimeService {
// tabs may exist only remotely), as are rows with no resolvable tabId
// (staleness unprovable). Session tabs count as existence: headless
// serve has no renderer graph, and session.tabs.list serves them.
continue
if (!src.restoredUnconfirmed || !src.restoredExactBinding) {
continue
}
}
const worktreeId = mirroredWorktreeId ?? src.worktreeId
if (!worktreeId) {
@@ -20566,83 +20561,142 @@ export class OrcaRuntimeService {
}
}
private inspectRestoredAgentPtyOnce(ptyId: string): Promise<RestoredAgentPtyInspection> | null {
const controller = this.ptyController
if (!controller?.inspectProcess) {
return null
}
const pending = this.restoredAgentPtyInspections.get(ptyId)
if (pending?.controller === controller) {
return pending.promise
}
const unavailable: RestoredAgentPtyInspection = {
controller,
foregroundProcess: null,
hasChildProcesses: false,
unavailable: true
}
let entry: RestoredAgentPtyInspectionEntry
const promise = Promise.resolve()
.then(() => controller.inspectProcess!(ptyId))
.then((inspection) => ({
controller,
foregroundProcess: inspection.foregroundProcess,
hasChildProcesses: inspection.hasChildProcesses,
unavailable: inspection.unavailable === true
}))
.catch(() => unavailable)
.finally(() => {
if (this.restoredAgentPtyInspections.get(ptyId) === entry) {
this.restoredAgentPtyInspections.delete(ptyId)
}
})
entry = { controller, promise }
this.restoredAgentPtyInspections.set(ptyId, entry)
return promise
}
private async resolveRestoredAgentLiveness(
entries: readonly AgentStatusIpcPayload[]
): Promise<Map<string, RestoredAgentLiveness>> {
entries: readonly AgentStatusIpcPayload[],
deadline: number
): Promise<Map<string, RestoredAgentLivenessResolution>> {
const restoredEntries = entries.filter((entry) => entry.restoredUnconfirmed === true)
const resolved = await Promise.all(
restoredEntries.map(async (entry): Promise<[string, RestoredAgentLiveness]> => {
const pty = this.getPtyRecordForPaneKey(entry.paneKey)
if (!pty || this.isPtyKnownExited(pty.ptyId)) {
return [entry.paneKey, 'exited']
}
const inspectionRead = this.inspectRestoredAgentPtyOnce(pty.ptyId)
if (!inspectionRead) {
return [entry.paneKey, 'unverifiable']
}
const inspection = await inspectionRead
const currentPty = this.getPtyRecordForPaneKey(entry.paneKey)
if (
inspection.controller !== this.ptyController ||
currentPty?.ptyId !== pty.ptyId ||
currentPty.incarnationId !== pty.incarnationId ||
inspection.unavailable
) {
return [entry.paneKey, 'unverifiable']
}
const recognized = recognizeAgentProcess(inspection.foregroundProcess)
if (recognized) {
return [
entry.paneKey,
isTuiAgent(entry.agentType) && recognized.agent !== entry.agentType ? 'exited' : 'live'
]
}
return [
restoredEntries.map(async (entry): Promise<[string, RestoredAgentLivenessResolution]> => {
const hookIdentity = restoredAgentHookIdentity(entry)
const binding = this.getRestoredAgentPtyBinding(entry)
const result = (
liveness: RestoredAgentLiveness,
hasExactBinding = binding !== null
): [string, RestoredAgentLivenessResolution] => [
entry.paneKey,
inspection.foregroundProcess !== null && !inspection.hasChildProcesses
{ liveness, hookIdentity, hasExactBinding }
]
if (!binding) {
return result('unverifiable', false)
}
const currentAtStart = this.getPtyRecordForPaneKey(entry.paneKey)
if (
currentAtStart &&
(currentAtStart.ptyId !== binding.ptyId ||
(binding.incarnationId !== null &&
currentAtStart.incarnationId !== binding.incarnationId))
) {
return result('exited')
}
if (currentAtStart && this.isPtyKnownExited(currentAtStart.ptyId)) {
return result('exited')
}
const controller = this.ptyController
if (!controller) {
return result('unverifiable')
}
const evidence = await this.ptyProcessLiveness.inspect({
source: controller,
ptyId: binding.ptyId,
identity:
binding.incarnationId ?? `runtime:${this.runtimeId}:${binding.lifecycleGeneration}`,
deadline
})
if (
controller !== this.ptyController ||
!this.isRestoredAgentPtyBindingCurrent(entry, binding)
) {
return result('unverifiable')
}
if (evidence.status !== 'live') {
return result(evidence.status)
}
if (binding.source === 'persisted') {
return result('unverifiable')
}
const recognized = recognizeAgentProcess(evidence.foregroundProcess)
if (recognized) {
return result(
isTuiAgent(entry.agentType) && recognized.agent !== entry.agentType ? 'exited' : 'live'
)
}
return result(
evidence.foregroundProcess !== null && evidence.hasChildProcesses === false
? 'exited'
: 'unverifiable'
]
)
})
)
return new Map(resolved)
}
private getRestoredAgentPtyBinding(entry: AgentStatusIpcPayload): RestoredAgentPtyBinding | null {
const persisted = this.getPersistedRestoredAgentPtyBinding(entry)
const current = this.getPtyRecordForPaneKey(entry.paneKey)
if (persisted && current) {
return current.ptyId === persisted.ptyId && current.incarnationId === persisted.incarnationId
? {
ptyId: current.ptyId,
incarnationId: current.incarnationId,
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
source: 'current'
}
: persisted
}
if (current) {
return {
ptyId: current.ptyId,
incarnationId: current.incarnationId,
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
source: 'current'
}
}
return persisted
}
private getPersistedRestoredAgentPtyBinding(
entry: AgentStatusIpcPayload
): RestoredAgentPtyBinding | null {
if (!entry.worktreeId) {
return null
}
const session = this.getWorkspaceSessionForWorktree(entry.worktreeId)
for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) {
if (
binding.paneKey === entry.paneKey &&
runtimeWorktreeIdsEqual(binding.worktreeId, entry.worktreeId)
) {
return {
ptyId,
incarnationId: binding.incarnationId,
lifecycleGeneration: this.getPtyLifecycleGeneration(ptyId),
source: 'persisted'
}
}
}
return null
}
private isRestoredAgentPtyBindingCurrent(
entry: AgentStatusIpcPayload,
binding: RestoredAgentPtyBinding
): boolean {
if (this.getPtyLifecycleGeneration(binding.ptyId) !== binding.lifecycleGeneration) {
return false
}
const current = this.getPtyRecordForPaneKey(entry.paneKey)
if (binding.source === 'current') {
return current?.ptyId === binding.ptyId && current.incarnationId === binding.incarnationId
}
const persisted = this.getPersistedRestoredAgentPtyBinding(entry)
return (
current === null &&
persisted?.ptyId === binding.ptyId &&
persisted.incarnationId === binding.incarnationId
)
}
listRepos(): Repo[] {
return this.store?.getRepos() ?? []
}
@@ -32525,6 +32579,7 @@ export class OrcaRuntimeService {
if (state.incarnationId !== undefined) {
if (pty.incarnationId && state.incarnationId && pty.incarnationId !== state.incarnationId) {
this.invalidatePtyIncarnationHandle(ptyId)
this.advancePtyLifecycleGeneration(ptyId)
}
pty.incarnationId = state.incarnationId
}
@@ -32556,6 +32611,9 @@ export class OrcaRuntimeService {
pty.paneKey = state.paneKey
}
if (state.connected !== undefined) {
if (state.connected && !pty.connected) {
this.ptyProcessLiveness.invalidate(ptyId)
}
pty.connected = state.connected
pty.disconnectedAt = state.connected ? null : (pty.disconnectedAt ?? Date.now())
}
@@ -38813,6 +38871,7 @@ export function resolveWorktreeScanCacheTtlMs(repo: Pick<Repo, 'path' | 'connect
: WORKTREE_SCAN_CACHE_TTL_MS
}
const PTY_CONTROLLER_LIST_TIMEOUT_MS = 3000
const PTY_PROCESS_INSPECTION_TIMEOUT_MS = 1_000
// Why: the slice of the list budget reserved for the aggregate to collect the providers
// that answered after a stalled one gives up.
const PTY_CONTROLLER_LIST_PROVIDER_MARGIN_MS = 500
@@ -40635,6 +40694,26 @@ function inferWorktreeIdFromPtyId(ptyId: string): string | null {
return parsePtySessionId(ptyId).worktreeId
}
function restoredAgentHookIdentity(entry: AgentStatusIpcPayload): string {
return JSON.stringify([
entry.paneKey,
entry.receivedAt,
entry.stateStartedAt,
entry.state,
entry.agentType,
entry.connectionId,
entry.worktreeId,
entry.tabId,
entry.terminalHandle,
entry.launchToken,
entry.prompt,
entry.lastAssistantMessage,
entry.toolName,
entry.toolInput,
entry.interrupted
])
}
function indexPersistedPtyWorktreeBindings(
session: WorkspaceSessionState | null | undefined
): ReadonlyMap<string, string> {
@@ -0,0 +1,134 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { PtyProcessLivenessBroker } from './pty-process-liveness-broker'
afterEach(() => {
vi.useRealTimers()
})
describe('PtyProcessLivenessBroker', () => {
it('bounds a hanging probe and does not fan out another unresolved inspection', async () => {
vi.useFakeTimers()
let resolveInspection!: (value: {
foregroundProcess: string | null
hasChildProcesses: boolean
}) => void
const inspection = new Promise<{
foregroundProcess: string | null
hasChildProcesses: boolean
}>((resolve) => {
resolveInspection = resolve
})
const inspectProcess = vi.fn(() => inspection)
const source = {
getForegroundProcess: vi.fn(async () => null),
inspectProcess
}
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 })
const first = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
await vi.advanceTimersByTimeAsync(100)
await expect(first).resolves.toEqual({
status: 'unverifiable',
reason: 'process inspection timed out'
})
await expect(
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
).resolves.toEqual({ status: 'unverifiable', reason: 'process inspection timed out' })
expect(inspectProcess).toHaveBeenCalledOnce()
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
await vi.runAllTimersAsync()
await expect(
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
).resolves.toEqual({
status: 'live',
foregroundProcess: 'codex',
hasChildProcesses: true
})
expect(inspectProcess).toHaveBeenCalledOnce()
})
it('caches live evidence across sequential clients', async () => {
const inspectProcess = vi.fn(async () => ({
foregroundProcess: 'codex',
hasChildProcesses: true
}))
const source = {
getForegroundProcess: vi.fn(async () => null),
inspectProcess
}
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 })
await broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
await broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
expect(inspectProcess).toHaveBeenCalledOnce()
})
it('treats only exact terminal_gone evidence as exited', async () => {
const gone = vi.fn(async () => {
throw new Error('terminal_gone')
})
const unavailable = vi.fn(async () => {
throw new Error('socket_closed')
})
const onInspectionError = vi.fn()
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100, onInspectionError })
await expect(
broker.inspect({
source: { getForegroundProcess: vi.fn(async () => null), inspectProcess: gone },
ptyId: 'pty-gone',
identity: 'incarnation-a'
})
).resolves.toEqual({ status: 'exited' })
await expect(
broker.inspect({
source: { getForegroundProcess: vi.fn(async () => null), inspectProcess: unavailable },
ptyId: 'pty-unavailable',
identity: 'incarnation-b'
})
).resolves.toEqual({ status: 'unverifiable', reason: 'socket_closed' })
expect(onInspectionError).toHaveBeenCalledExactlyOnceWith(
'pty-unavailable',
expect.objectContaining({ message: 'socket_closed' })
)
})
it('does not let an old incarnation populate the replacement cache entry', async () => {
let resolveOld!: (value: { foregroundProcess: string; hasChildProcesses: boolean }) => void
const oldInspection = new Promise<{
foregroundProcess: string
hasChildProcesses: boolean
}>((resolve) => {
resolveOld = resolve
})
const inspectProcess = vi
.fn()
.mockReturnValueOnce(oldInspection)
.mockResolvedValueOnce({ foregroundProcess: 'zsh', hasChildProcesses: false })
const source = {
getForegroundProcess: vi.fn(async () => null),
inspectProcess
}
const broker = new PtyProcessLivenessBroker({ timeoutMs: 1_000 })
const oldRead = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
const replacementRead = broker.inspect({
source,
ptyId: 'pty-1',
identity: 'incarnation-b'
})
resolveOld({ foregroundProcess: 'codex', hasChildProcesses: true })
await expect(oldRead).resolves.toMatchObject({ status: 'live', foregroundProcess: 'codex' })
await expect(replacementRead).resolves.toMatchObject({
status: 'live',
foregroundProcess: 'zsh'
})
await expect(
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-b' })
).resolves.toMatchObject({ status: 'live', foregroundProcess: 'zsh' })
expect(inspectProcess).toHaveBeenCalledTimes(2)
})
})
@@ -0,0 +1,250 @@
export type PtyProcessInspectionSource = {
getForegroundProcess(ptyId: string): Promise<string | null>
inspectProcess?: (ptyId: string) => Promise<{
foregroundProcess: string | null
hasChildProcesses: boolean
unavailable?: true
}>
hasChildProcesses?: (ptyId: string) => Promise<boolean>
}
export type PtyProcessLivenessEvidence =
| {
status: 'live'
foregroundProcess: string | null
hasChildProcesses: boolean | null
}
| { status: 'unverifiable'; reason: string }
| { status: 'exited' }
type PtyProcessEvidenceEntry = {
source: PtyProcessInspectionSource
identity: string
freshness: number
failureCount: number
evidence: PtyProcessLivenessEvidence | null
expiresAt: number
timedOut: boolean
probe: Promise<PtyProcessLivenessEvidence> | null
}
export type PtyProcessLivenessBrokerOptions = {
timeoutMs: number
liveTtlMs?: number
unavailableBackoffBaseMs?: number
unavailableBackoffMaxMs?: number
now?: () => number
onInspectionError?: (ptyId: string, error: unknown) => void
}
const DEFAULT_LIVE_TTL_MS = 10_000
const DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS = 3_000
const DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS = 30_000
export class PtyProcessLivenessBroker {
private readonly entries = new Map<string, PtyProcessEvidenceEntry>()
private readonly now: () => number
constructor(private readonly options: PtyProcessLivenessBrokerOptions) {
this.now = options.now ?? Date.now
}
inspect(args: {
source: PtyProcessInspectionSource
ptyId: string
identity: string
freshness?: number
reuseSettled?: boolean
deadline?: number
}): Promise<PtyProcessLivenessEvidence> {
const freshness = args.freshness ?? 0
const waitMs = Math.max(
0,
Math.min(
this.options.timeoutMs,
args.deadline === undefined ? this.options.timeoutMs : args.deadline - this.now()
)
)
const existing = this.entries.get(args.ptyId)
if (
existing?.source === args.source &&
existing.identity === args.identity &&
existing.probe &&
existing.freshness < freshness
) {
if (waitMs === 0) {
return Promise.resolve({
status: 'unverifiable',
reason: 'process inspection timed out'
})
}
return this.waitForProbe(args.ptyId, existing, waitMs).then(() =>
existing.probe
? { status: 'unverifiable', reason: 'process inspection timed out' }
: this.inspect(args)
)
}
if (
existing?.source === args.source &&
existing.identity === args.identity &&
existing.freshness >= freshness
) {
if (existing.probe) {
return existing.timedOut
? Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
: waitMs === 0
? Promise.resolve({
status: 'unverifiable',
reason: 'process inspection timed out'
})
: this.waitForProbe(args.ptyId, existing, waitMs)
}
if (args.reuseSettled !== false && existing.evidence && existing.expiresAt > this.now()) {
return Promise.resolve(existing.evidence)
}
}
if (waitMs === 0) {
return Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
}
const failureCount =
existing?.source === args.source && existing.identity === args.identity
? existing.failureCount
: 0
const entry: PtyProcessEvidenceEntry = {
source: args.source,
identity: args.identity,
freshness,
failureCount,
evidence: null,
expiresAt: 0,
timedOut: false,
probe: null
}
const probe = this.runProbe(args.source, args.ptyId)
.then((evidence) => {
if (this.entries.get(args.ptyId) !== entry) {
return evidence
}
entry.probe = null
entry.timedOut = false
entry.evidence = evidence
if (evidence.status === 'unverifiable') {
entry.failureCount += 1
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
} else if (evidence.status === 'live') {
entry.failureCount = 0
entry.expiresAt = this.now() + (this.options.liveTtlMs ?? DEFAULT_LIVE_TTL_MS)
} else {
entry.failureCount = 0
entry.expiresAt = Number.POSITIVE_INFINITY
}
return evidence
})
.catch((error): PtyProcessLivenessEvidence => {
if (this.entries.get(args.ptyId) === entry) {
entry.probe = null
entry.timedOut = false
entry.failureCount += 1
entry.evidence = { status: 'unverifiable', reason: describeError(error) }
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
}
this.options.onInspectionError?.(args.ptyId, error)
return { status: 'unverifiable', reason: describeError(error) }
})
entry.probe = probe
this.entries.set(args.ptyId, entry)
return this.waitForProbe(args.ptyId, entry, waitMs)
}
invalidate(ptyId: string): void {
this.entries.delete(ptyId)
}
invalidateAll(): void {
this.entries.clear()
}
getPendingCount(): number {
let count = 0
for (const entry of this.entries.values()) {
if (entry.probe) {
count += 1
}
}
return count
}
private async runProbe(
source: PtyProcessInspectionSource,
ptyId: string
): Promise<PtyProcessLivenessEvidence> {
try {
if (source.inspectProcess) {
const inspection = await source.inspectProcess(ptyId)
return inspection.unavailable
? { status: 'unverifiable', reason: 'process inspection unavailable' }
: {
status: 'live',
foregroundProcess: inspection.foregroundProcess,
hasChildProcesses: inspection.hasChildProcesses
}
}
const foregroundProcess = await source.getForegroundProcess(ptyId)
return {
status: 'live',
foregroundProcess,
hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null
}
} catch (error) {
if (isTerminalGoneError(error)) {
return { status: 'exited' }
}
throw error
}
}
private waitForProbe(
ptyId: string,
entry: PtyProcessEvidenceEntry,
timeoutMs: number
): Promise<PtyProcessLivenessEvidence> {
const probe = entry.probe
if (!probe) {
return Promise.resolve(
entry.evidence ?? { status: 'unverifiable', reason: 'process inspection unavailable' }
)
}
return new Promise((resolve) => {
const timeout = setTimeout(() => {
if (this.entries.get(ptyId) === entry && entry.probe === probe) {
entry.timedOut = true
}
resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
}, timeoutMs)
void probe.then((evidence) => {
clearTimeout(timeout)
resolve(evidence)
})
})
}
private unavailableBackoffMs(failureCount: number): number {
const base = this.options.unavailableBackoffBaseMs ?? DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS
const max = this.options.unavailableBackoffMaxMs ?? DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS
return Math.min(max, base * 2 ** Math.max(0, failureCount - 1))
}
}
function isTerminalGoneError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error)
const code =
error && typeof error === 'object' && 'code' in error
? String((error as { code?: unknown }).code)
: ''
return message === 'terminal_gone' || code === 'terminal_gone'
}
function describeError(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}