mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 16:03:49 +00:00
fix(runtime): fence restored agent liveness evidence
This commit is contained in:
@@ -10293,7 +10293,7 @@
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
|
||||
"cd mobile && pnpm exec vitest run src/worktree/agent-row-display.test.ts src/transport/stable-logical-rpc-client.test.ts",
|
||||
"pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1",
|
||||
"pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1",
|
||||
@@ -10305,6 +10305,7 @@
|
||||
"src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts",
|
||||
"src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts",
|
||||
"src/main/runtime/orca-runtime.test.ts",
|
||||
"src/main/runtime/pty-process-liveness-broker.test.ts",
|
||||
"src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts",
|
||||
"src/cli/handlers/orchestration.test.ts",
|
||||
"src/cli/handlers/orchestration-lifecycle-rejection.test.ts",
|
||||
@@ -10356,7 +10357,10 @@
|
||||
"retained renderer output remains readable through the recovered terminal",
|
||||
"capable worktree inventory projects restored live and unverifiable identities but omits exited shell-only agents",
|
||||
"a connected live PTY remains inventoried when its user-visible tab is missing",
|
||||
"legacy clients retain the exact restored-row suppression without a process inspection"
|
||||
"legacy clients retain the exact restored-row suppression",
|
||||
"inspection timeout, transport loss, missing bindings, and stale incarnations remain unverifiable",
|
||||
"terminal_gone is the only inspection rejection that retires the exact restored row",
|
||||
"sequential inventory clients reuse bounded process evidence instead of polling each restored row"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -10504,10 +10508,10 @@
|
||||
"date": "2026-08-23",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 15.33,
|
||||
"summary": "Seven files passed 1,264 tests with one skip, covering live, unverifiable, exited, hidden-tab, legacy and request-level capability projection plus every encrypted desktop/CLI client capability advertisement. Mobile typecheck and 25 focused mobile tests separately passed."
|
||||
"summary": "Eight focused runtime and wire files passed 1,273 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, cold persisted binding, incarnation and hook replacement races, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, and encrypted desktop/CLI capability advertisement. Mobile typecheck and 25 focused mobile tests passed separately."
|
||||
},
|
||||
{
|
||||
"date": "2026-08-13",
|
||||
@@ -10686,7 +10690,7 @@
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. Capable worktree.ps requests inspect only restored-unconfirmed candidates, deduplicate concurrent and same-request reads by PTY, and do no inspection for legacy clients, confirmed rows, unrelated PTYs, or restored rows without a retained PTY. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path."
|
||||
"evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. One incarnation-scoped process-evidence broker now serves both the incumbent foreground-agent refresh and capable restored-row projection: concurrent callers deduplicate, sequential live reads reuse a 10-second TTL, unavailable reads back off from 3 to 30 seconds, a hung provider retains at most one unresolved probe per identity, and every catalog read shares the existing three-second controller budget. Lifecycle, incarnation, reconnect, and controller changes invalidate evidence. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federation replay keeps the existing per-Dispatch polling cadence and existing transport queue limit, retaining a terminal Dispatch only while its imported cursor exceeds one persisted acknowledgment watermark. Each successful ACK adds one primary-key SQLite update; exact eligibility is a primary-key lookup, and identical pending terminal lookup is bounded by the existing 256-item per-Dispatch relay quota and pending-relay index. A version-aware schema sentinel keeps valid v26 databases on the direct v26-to-v27 migration path."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Collect 100 consecutive passes or 14 days of stable CI history on macOS, Linux, and Windows.",
|
||||
|
||||
@@ -22,7 +22,10 @@ import type { WorkspaceSessionState } from '../../shared/workspace-session-state
|
||||
import type { WorkspaceLineage, WorktreeLineage } from '../../shared/worktree/lineage-types'
|
||||
import type { WorktreeMeta } from '../../shared/worktree/meta-types'
|
||||
import type { Worktree } from '../../shared/worktree/types'
|
||||
import { AGENT_STATUS_STALE_AFTER_MS } from '../../shared/agent-status-types'
|
||||
import {
|
||||
AGENT_STATUS_STALE_AFTER_MS,
|
||||
type AgentStatusIpcPayload
|
||||
} from '../../shared/agent-status-types'
|
||||
import {
|
||||
reviewHeadRemoteRefComponent,
|
||||
REVIEW_HEAD_FETCH_TIMEOUT_MS
|
||||
@@ -9873,12 +9876,12 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(
|
||||
(
|
||||
runtime as unknown as {
|
||||
ptyForegroundProcessReads: Map<string, unknown>
|
||||
ptyProcessLiveness: { getPendingCount(): number }
|
||||
}
|
||||
).ptyForegroundProcessReads.size
|
||||
).ptyProcessLiveness.getPendingCount()
|
||||
).toBe(0)
|
||||
)
|
||||
await Promise.resolve()
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
|
||||
getForegroundProcess.mockResolvedValueOnce('zsh')
|
||||
runtime.onPtyData('pty-1', '\x1b]0;other cwd\x07', 102)
|
||||
@@ -37744,12 +37747,146 @@ describe('OrcaRuntimeService', () => {
|
||||
leaves: []
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs()
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, false)
|
||||
const summary = worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)
|
||||
|
||||
expect(summary).toMatchObject({ hasHostSidebarActivity: false, status: 'inactive', agents: [] })
|
||||
})
|
||||
|
||||
it('keeps a restored row unverifiable when current PTY bookkeeping is missing', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
const now = Date.now()
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'tab-1',
|
||||
state: 'working',
|
||||
prompt: 'await provider adoption',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Codex',
|
||||
activeLeafId: leafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: []
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('uses a cold persisted binding without certifying a replacement incarnation live', async () => {
|
||||
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
|
||||
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
const session = makeWorkspaceSessionWithHeadlessTerminal({
|
||||
terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId }
|
||||
})
|
||||
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const now = Date.now()
|
||||
const inspectProcess = vi.fn(async () => ({
|
||||
foregroundProcess: 'codex',
|
||||
hasChildProcesses: true
|
||||
}))
|
||||
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'host-tab',
|
||||
state: 'working',
|
||||
prompt: 'cold persisted agent',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => 'codex'),
|
||||
inspectProcess,
|
||||
listProcesses: vi.fn(async () => [])
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('persisted-pty')
|
||||
})
|
||||
|
||||
it('omits a cold restored row only after terminal_gone confirms exact absence', async () => {
|
||||
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
|
||||
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
const session = makeWorkspaceSessionWithHeadlessTerminal({
|
||||
terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId }
|
||||
})
|
||||
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const now = Date.now()
|
||||
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'host-tab',
|
||||
state: 'working',
|
||||
prompt: 'completed while runtime was down',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess: vi.fn(async () => {
|
||||
throw new Error('terminal_gone')
|
||||
}),
|
||||
listProcesses: vi.fn(async () => [])
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
|
||||
[]
|
||||
)
|
||||
})
|
||||
|
||||
it('projects a restored agent after the execution host confirms its exact PTY is live', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
@@ -37833,7 +37970,7 @@ describe('OrcaRuntimeService', () => {
|
||||
agentLiveness: 'live'
|
||||
})
|
||||
])
|
||||
expect(inspectProcess).toHaveBeenCalledTimes(2)
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('keeps a restored agent visible as unverifiable when PTY inspection loses contact', async () => {
|
||||
@@ -37900,6 +38037,161 @@ describe('OrcaRuntimeService', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('fences a restored-agent inspection across PTY reincarnation', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
const now = Date.now()
|
||||
let resolveInspection!: (value: {
|
||||
foregroundProcess: string
|
||||
hasChildProcesses: boolean
|
||||
}) => void
|
||||
const inspectProcess = vi.fn(
|
||||
() =>
|
||||
new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => {
|
||||
resolveInspection = resolve
|
||||
})
|
||||
)
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'tab-1',
|
||||
state: 'working',
|
||||
prompt: 'old incarnation',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => 'codex'),
|
||||
inspectProcess,
|
||||
listProcesses: vi.fn(async () => [])
|
||||
})
|
||||
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
incarnationId: 'incarnation-a'
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Codex',
|
||||
activeLeafId: leafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: 'pty-1'
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const inventory = runtime.getWorktreePs(undefined, true, true)
|
||||
await vi.waitFor(() => expect(inspectProcess).toHaveBeenCalledOnce())
|
||||
runtime.onPtySpawned('pty-1', 'incarnation-b', { awaitsRegistration: false })
|
||||
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
|
||||
|
||||
const { worktrees } = await inventory
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('does not publish a restored hook snapshot replaced during inspection', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
const now = Date.now()
|
||||
let rows: AgentStatusIpcPayload[] = [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'tab-1',
|
||||
state: 'working',
|
||||
prompt: 'stale hook',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
let resolveInspection!: (value: {
|
||||
foregroundProcess: string
|
||||
hasChildProcesses: boolean
|
||||
}) => void
|
||||
const inspectProcess = vi.fn(
|
||||
() =>
|
||||
new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>((resolve) => {
|
||||
resolveInspection = resolve
|
||||
})
|
||||
)
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => rows
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => 'codex'),
|
||||
inspectProcess,
|
||||
listProcesses: vi.fn(async () => [])
|
||||
})
|
||||
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
incarnationId: 'incarnation-a'
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Codex',
|
||||
activeLeafId: leafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: 'pty-1'
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const inventory = runtime.getWorktreePs(undefined, true, true)
|
||||
await vi.waitFor(() => expect(inspectProcess).toHaveBeenCalledOnce())
|
||||
rows = []
|
||||
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
|
||||
|
||||
const { worktrees } = await inventory
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
|
||||
[]
|
||||
)
|
||||
})
|
||||
|
||||
it('omits an exited restored agent after the exact PTY is confirmed shell-only', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
@@ -38018,7 +38310,7 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
|
||||
[]
|
||||
)
|
||||
expect(inspectProcess).not.toHaveBeenCalled()
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('uses mirrored tab ownership after a workspace rename instead of stale hook attribution', async () => {
|
||||
|
||||
+230
-151
@@ -660,6 +660,7 @@ import type { RuntimeBrowserCommands } from './orca-runtime-browser'
|
||||
import { createRuntimeBrowserCommands } from './runtime-browser-commands-factory'
|
||||
import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency'
|
||||
import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity'
|
||||
import { PtyProcessLivenessBroker } from './pty-process-liveness-broker'
|
||||
import {
|
||||
buildHeadlessTerminalSplitLayout,
|
||||
countTerminalLayoutLeaves,
|
||||
@@ -1631,26 +1632,21 @@ type PtyForegroundProcessRead = {
|
||||
available: boolean
|
||||
}
|
||||
|
||||
type PtyForegroundProcessReadEntry = {
|
||||
controller: RuntimePtyController
|
||||
startedAfterTitleObservation: number
|
||||
promise: Promise<PtyForegroundProcessRead>
|
||||
}
|
||||
|
||||
type RestoredAgentPtyInspection = {
|
||||
controller: RuntimePtyController
|
||||
foregroundProcess: string | null
|
||||
hasChildProcesses: boolean
|
||||
unavailable: boolean
|
||||
}
|
||||
|
||||
type RestoredAgentPtyInspectionEntry = {
|
||||
controller: RuntimePtyController
|
||||
promise: Promise<RestoredAgentPtyInspection>
|
||||
}
|
||||
|
||||
type RestoredAgentLiveness = 'live' | 'unverifiable' | 'exited'
|
||||
|
||||
type RestoredAgentLivenessResolution = {
|
||||
liveness: RestoredAgentLiveness
|
||||
hookIdentity: string
|
||||
hasExactBinding: boolean
|
||||
}
|
||||
|
||||
type RestoredAgentPtyBinding = {
|
||||
ptyId: string
|
||||
incarnationId: PtyIncarnationId | null
|
||||
lifecycleGeneration: number
|
||||
source: 'current' | 'persisted'
|
||||
}
|
||||
|
||||
function copySleepingAgentLaunchConfig(
|
||||
config: SleepingAgentLaunchConfig
|
||||
): SleepingAgentLaunchConfig {
|
||||
@@ -3184,8 +3180,12 @@ export class OrcaRuntimeService {
|
||||
private worktreeAdminFingerprintProbes = new Set<string>()
|
||||
private cloneInFlightByPath = new Map<string, Promise<void>>()
|
||||
private ptyForegroundAgentRefreshes = new Map<string, PtyForegroundAgentRefresh>()
|
||||
private ptyForegroundProcessReads = new Map<string, PtyForegroundProcessReadEntry>()
|
||||
private restoredAgentPtyInspections = new Map<string, RestoredAgentPtyInspectionEntry>()
|
||||
private readonly ptyProcessLiveness = new PtyProcessLivenessBroker({
|
||||
timeoutMs: PTY_PROCESS_INSPECTION_TIMEOUT_MS,
|
||||
onInspectionError: (ptyId, error) => {
|
||||
console.warn(`[runtime] PTY process inspection failed for ${ptyId}`, error)
|
||||
}
|
||||
})
|
||||
private ptyDelayedForegroundSnapshotTitleObservations = new Map<string, number>()
|
||||
// Why a set and not a timer: the intent is retired by the exit it explains, or
|
||||
// by the next lifecycle generation on that id (advancePtyLifecycleGeneration),
|
||||
@@ -6181,7 +6181,10 @@ export class OrcaRuntimeService {
|
||||
// Why: CLI terminal writes must go through the main-owned PTY registry
|
||||
// instead of tunneling back through renderer IPC, or live handles could
|
||||
// drift from the process they are supposed to control during reloads.
|
||||
this.ptyController = controller
|
||||
if (this.ptyController !== controller) {
|
||||
this.ptyProcessLiveness.invalidateAll()
|
||||
this.ptyController = controller
|
||||
}
|
||||
}
|
||||
|
||||
setNotifier(notifier: RuntimeNotifier | null): void {
|
||||
@@ -10795,6 +10798,12 @@ export class OrcaRuntimeService {
|
||||
incarnationId?: PtyIncarnationId,
|
||||
options: { awaitsRegistration?: boolean } = {}
|
||||
): void {
|
||||
const previousIncarnation = this.ptysById.get(ptyId)?.incarnationId
|
||||
if (previousIncarnation && incarnationId && previousIncarnation !== incarnationId) {
|
||||
this.advancePtyLifecycleGeneration(ptyId)
|
||||
} else {
|
||||
this.ptyProcessLiveness.invalidate(ptyId)
|
||||
}
|
||||
this.forgetPtyLivenessVerdict(ptyId)
|
||||
if (options.awaitsRegistration !== false) {
|
||||
// Why: surface absence cannot distinguish an in-flight admission from a completed headless lifecycle.
|
||||
@@ -12262,6 +12271,7 @@ export class OrcaRuntimeService {
|
||||
|
||||
private advancePtyLifecycleGeneration(ptyId: string): void {
|
||||
this.ptyLifecycleGenerationById.set(ptyId, this.nextPtyLifecycleGeneration++)
|
||||
this.ptyProcessLiveness.invalidate(ptyId)
|
||||
// Why: a stop whose exit never arrived would otherwise stay armed across a
|
||||
// same-id respawn and label the NEXT process's crash an operator close —
|
||||
// the exact lie this cause model exists to remove.
|
||||
@@ -19050,73 +19060,44 @@ export class OrcaRuntimeService {
|
||||
|
||||
private readPtyForegroundProcessFromController(
|
||||
ptyId: string,
|
||||
afterTitleObservation = 0
|
||||
afterTitleObservation = 0,
|
||||
options: { reuseSettled?: boolean } = {}
|
||||
): Promise<PtyForegroundProcessRead> | null {
|
||||
const controller = this.ptyController
|
||||
if (!controller) {
|
||||
return null
|
||||
}
|
||||
const pending = this.ptyForegroundProcessReads.get(ptyId)
|
||||
if (
|
||||
pending?.controller === controller &&
|
||||
pending.startedAfterTitleObservation >= afterTitleObservation
|
||||
) {
|
||||
return pending.promise
|
||||
}
|
||||
if (pending?.controller === controller) {
|
||||
return pending.promise.then(
|
||||
() =>
|
||||
this.readPtyForegroundProcessFromController(ptyId, afterTitleObservation) ?? {
|
||||
controller,
|
||||
process: null,
|
||||
available: false
|
||||
}
|
||||
)
|
||||
}
|
||||
const unavailable: PtyForegroundProcessRead = {
|
||||
controller,
|
||||
process: null,
|
||||
available: false
|
||||
}
|
||||
let processRead: Promise<string | null>
|
||||
try {
|
||||
processRead = Promise.resolve(controller.getForegroundProcess(ptyId))
|
||||
} catch {
|
||||
const entry: PtyForegroundProcessReadEntry = {
|
||||
const identity = this.getPtyProcessEvidenceIdentity(ptyId)
|
||||
return this.ptyProcessLiveness
|
||||
.inspect({
|
||||
source: controller,
|
||||
ptyId,
|
||||
identity,
|
||||
freshness: afterTitleObservation,
|
||||
reuseSettled: options.reuseSettled
|
||||
})
|
||||
.then((evidence) => ({
|
||||
controller,
|
||||
startedAfterTitleObservation: afterTitleObservation,
|
||||
promise: Promise.resolve(unavailable)
|
||||
}
|
||||
entry.promise = entry.promise.finally(() => {
|
||||
if (this.ptyForegroundProcessReads.get(ptyId) === entry) {
|
||||
this.ptyForegroundProcessReads.delete(ptyId)
|
||||
}
|
||||
})
|
||||
this.ptyForegroundProcessReads.set(ptyId, entry)
|
||||
return entry.promise
|
||||
}
|
||||
let entry: PtyForegroundProcessReadEntry
|
||||
const promise = processRead
|
||||
.then((process) => ({ controller, process, available: true }))
|
||||
.catch(() => unavailable)
|
||||
.finally(() => {
|
||||
if (this.ptyForegroundProcessReads.get(ptyId) === entry) {
|
||||
this.ptyForegroundProcessReads.delete(ptyId)
|
||||
}
|
||||
})
|
||||
entry = {
|
||||
controller,
|
||||
startedAfterTitleObservation: afterTitleObservation,
|
||||
promise
|
||||
}
|
||||
this.ptyForegroundProcessReads.set(ptyId, entry)
|
||||
return entry.promise
|
||||
process: evidence.status === 'live' ? evidence.foregroundProcess : null,
|
||||
available: evidence.status === 'live'
|
||||
}))
|
||||
}
|
||||
|
||||
private getPtyProcessEvidenceIdentity(ptyId: string): string {
|
||||
return (
|
||||
this.ptysById.get(ptyId)?.incarnationId ??
|
||||
`runtime:${this.runtimeId}:${this.getPtyLifecycleGeneration(ptyId)}`
|
||||
)
|
||||
}
|
||||
|
||||
private confirmPtyAgentExit(ptyId: string): void {
|
||||
const pty = this.ptysById.get(ptyId)
|
||||
const titleObservedAt = pty?.lastOscTitleAt ?? null
|
||||
const foregroundRead = this.readPtyForegroundProcessFromController(ptyId, titleObservedAt ?? 0)
|
||||
const foregroundRead = this.readPtyForegroundProcessFromController(
|
||||
ptyId,
|
||||
titleObservedAt ?? 0,
|
||||
{ reuseSettled: false }
|
||||
)
|
||||
if (!pty?.connected || !foregroundRead) {
|
||||
this.recordTerminalSideEffectFact(ptyId, { kind: 'agent-exited' })
|
||||
return
|
||||
@@ -19996,6 +19977,7 @@ export class OrcaRuntimeService {
|
||||
visibilitySettings
|
||||
)
|
||||
)
|
||||
const ptyProcessEvidenceDeadline = Date.now() + PTY_CONTROLLER_LIST_TIMEOUT_MS
|
||||
// Why: worktree.ps backs the mobile sidebar, so it must use the same
|
||||
// host-owned imported-worktree visibility gate as worktree.list/desktop.
|
||||
const freshPtyLiveness = await this.refreshPtyWorktreeRecordsFromController(resolvedWorktrees)
|
||||
@@ -20355,7 +20337,8 @@ export class OrcaRuntimeService {
|
||||
missingRuntimeWorktreeIds,
|
||||
mirroredWorktreeIdByTabId,
|
||||
connectedPtyEvidence,
|
||||
restoredAgentPresenceSupported
|
||||
restoredAgentPresenceSupported,
|
||||
ptyProcessEvidenceDeadline
|
||||
)
|
||||
|
||||
const sorted = [...summaries.values()].sort(compareWorktreePs)
|
||||
@@ -20380,7 +20363,8 @@ export class OrcaRuntimeService {
|
||||
paneKeys: ReadonlySet<string>
|
||||
ptyIds: ReadonlySet<string>
|
||||
},
|
||||
restoredAgentPresenceSupported: boolean
|
||||
restoredAgentPresenceSupported: boolean,
|
||||
ptyProcessEvidenceDeadline: number
|
||||
): Promise<void> {
|
||||
// Why: most agents report via hooks (agent-hooks/server), not OSC, so the
|
||||
// hook snapshot is the primary source — same one the desktop sidebar reads.
|
||||
@@ -20404,6 +20388,7 @@ export class OrcaRuntimeService {
|
||||
updatedAt: number
|
||||
restoredUnconfirmed?: boolean
|
||||
agentLiveness?: Exclude<RestoredAgentLiveness, 'exited'>
|
||||
restoredExactBinding?: boolean
|
||||
}
|
||||
>()
|
||||
for (const snapshot of this.latestAgentStatusByPaneKey.values()) {
|
||||
@@ -20425,13 +20410,20 @@ export class OrcaRuntimeService {
|
||||
updatedAt: snapshot.updatedAt
|
||||
})
|
||||
}
|
||||
const hookEntries = this.getAgentStatusSnapshotFn?.() ?? []
|
||||
const hookEntriesAtInspection = this.getAgentStatusSnapshotFn?.() ?? []
|
||||
const restoredAgentLiveness = restoredAgentPresenceSupported
|
||||
? await this.resolveRestoredAgentLiveness(hookEntries)
|
||||
: new Map<string, RestoredAgentLiveness>()
|
||||
? await this.resolveRestoredAgentLiveness(hookEntriesAtInspection, ptyProcessEvidenceDeadline)
|
||||
: new Map<string, RestoredAgentLivenessResolution>()
|
||||
const hookEntries = restoredAgentPresenceSupported
|
||||
? (this.getAgentStatusSnapshotFn?.() ?? hookEntriesAtInspection)
|
||||
: hookEntriesAtInspection
|
||||
for (const entry of hookEntries) {
|
||||
const liveness =
|
||||
const livenessResolution =
|
||||
entry.restoredUnconfirmed === true ? restoredAgentLiveness.get(entry.paneKey) : undefined
|
||||
const liveness =
|
||||
livenessResolution?.hookIdentity === restoredAgentHookIdentity(entry)
|
||||
? livenessResolution.liveness
|
||||
: undefined
|
||||
// Why: legacy clients ignore restored provenance, while a confirmed shell proves the old agent exited.
|
||||
if (
|
||||
entry.restoredUnconfirmed === true &&
|
||||
@@ -20465,7 +20457,8 @@ export class OrcaRuntimeService {
|
||||
...(entry.restoredUnconfirmed === true
|
||||
? {
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: liveness as Exclude<RestoredAgentLiveness, 'exited'>
|
||||
agentLiveness: liveness as Exclude<RestoredAgentLiveness, 'exited'>,
|
||||
restoredExactBinding: livenessResolution?.hasExactBinding === true
|
||||
}
|
||||
: {})
|
||||
})
|
||||
@@ -20501,7 +20494,9 @@ export class OrcaRuntimeService {
|
||||
// tabs may exist only remotely), as are rows with no resolvable tabId
|
||||
// (staleness unprovable). Session tabs count as existence: headless
|
||||
// serve has no renderer graph, and session.tabs.list serves them.
|
||||
continue
|
||||
if (!src.restoredUnconfirmed || !src.restoredExactBinding) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
const worktreeId = mirroredWorktreeId ?? src.worktreeId
|
||||
if (!worktreeId) {
|
||||
@@ -20566,83 +20561,142 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
|
||||
private inspectRestoredAgentPtyOnce(ptyId: string): Promise<RestoredAgentPtyInspection> | null {
|
||||
const controller = this.ptyController
|
||||
if (!controller?.inspectProcess) {
|
||||
return null
|
||||
}
|
||||
const pending = this.restoredAgentPtyInspections.get(ptyId)
|
||||
if (pending?.controller === controller) {
|
||||
return pending.promise
|
||||
}
|
||||
const unavailable: RestoredAgentPtyInspection = {
|
||||
controller,
|
||||
foregroundProcess: null,
|
||||
hasChildProcesses: false,
|
||||
unavailable: true
|
||||
}
|
||||
let entry: RestoredAgentPtyInspectionEntry
|
||||
const promise = Promise.resolve()
|
||||
.then(() => controller.inspectProcess!(ptyId))
|
||||
.then((inspection) => ({
|
||||
controller,
|
||||
foregroundProcess: inspection.foregroundProcess,
|
||||
hasChildProcesses: inspection.hasChildProcesses,
|
||||
unavailable: inspection.unavailable === true
|
||||
}))
|
||||
.catch(() => unavailable)
|
||||
.finally(() => {
|
||||
if (this.restoredAgentPtyInspections.get(ptyId) === entry) {
|
||||
this.restoredAgentPtyInspections.delete(ptyId)
|
||||
}
|
||||
})
|
||||
entry = { controller, promise }
|
||||
this.restoredAgentPtyInspections.set(ptyId, entry)
|
||||
return promise
|
||||
}
|
||||
|
||||
private async resolveRestoredAgentLiveness(
|
||||
entries: readonly AgentStatusIpcPayload[]
|
||||
): Promise<Map<string, RestoredAgentLiveness>> {
|
||||
entries: readonly AgentStatusIpcPayload[],
|
||||
deadline: number
|
||||
): Promise<Map<string, RestoredAgentLivenessResolution>> {
|
||||
const restoredEntries = entries.filter((entry) => entry.restoredUnconfirmed === true)
|
||||
const resolved = await Promise.all(
|
||||
restoredEntries.map(async (entry): Promise<[string, RestoredAgentLiveness]> => {
|
||||
const pty = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (!pty || this.isPtyKnownExited(pty.ptyId)) {
|
||||
return [entry.paneKey, 'exited']
|
||||
}
|
||||
const inspectionRead = this.inspectRestoredAgentPtyOnce(pty.ptyId)
|
||||
if (!inspectionRead) {
|
||||
return [entry.paneKey, 'unverifiable']
|
||||
}
|
||||
const inspection = await inspectionRead
|
||||
const currentPty = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (
|
||||
inspection.controller !== this.ptyController ||
|
||||
currentPty?.ptyId !== pty.ptyId ||
|
||||
currentPty.incarnationId !== pty.incarnationId ||
|
||||
inspection.unavailable
|
||||
) {
|
||||
return [entry.paneKey, 'unverifiable']
|
||||
}
|
||||
const recognized = recognizeAgentProcess(inspection.foregroundProcess)
|
||||
if (recognized) {
|
||||
return [
|
||||
entry.paneKey,
|
||||
isTuiAgent(entry.agentType) && recognized.agent !== entry.agentType ? 'exited' : 'live'
|
||||
]
|
||||
}
|
||||
return [
|
||||
restoredEntries.map(async (entry): Promise<[string, RestoredAgentLivenessResolution]> => {
|
||||
const hookIdentity = restoredAgentHookIdentity(entry)
|
||||
const binding = this.getRestoredAgentPtyBinding(entry)
|
||||
const result = (
|
||||
liveness: RestoredAgentLiveness,
|
||||
hasExactBinding = binding !== null
|
||||
): [string, RestoredAgentLivenessResolution] => [
|
||||
entry.paneKey,
|
||||
inspection.foregroundProcess !== null && !inspection.hasChildProcesses
|
||||
{ liveness, hookIdentity, hasExactBinding }
|
||||
]
|
||||
if (!binding) {
|
||||
return result('unverifiable', false)
|
||||
}
|
||||
const currentAtStart = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (
|
||||
currentAtStart &&
|
||||
(currentAtStart.ptyId !== binding.ptyId ||
|
||||
(binding.incarnationId !== null &&
|
||||
currentAtStart.incarnationId !== binding.incarnationId))
|
||||
) {
|
||||
return result('exited')
|
||||
}
|
||||
if (currentAtStart && this.isPtyKnownExited(currentAtStart.ptyId)) {
|
||||
return result('exited')
|
||||
}
|
||||
const controller = this.ptyController
|
||||
if (!controller) {
|
||||
return result('unverifiable')
|
||||
}
|
||||
const evidence = await this.ptyProcessLiveness.inspect({
|
||||
source: controller,
|
||||
ptyId: binding.ptyId,
|
||||
identity:
|
||||
binding.incarnationId ?? `runtime:${this.runtimeId}:${binding.lifecycleGeneration}`,
|
||||
deadline
|
||||
})
|
||||
if (
|
||||
controller !== this.ptyController ||
|
||||
!this.isRestoredAgentPtyBindingCurrent(entry, binding)
|
||||
) {
|
||||
return result('unverifiable')
|
||||
}
|
||||
if (evidence.status !== 'live') {
|
||||
return result(evidence.status)
|
||||
}
|
||||
if (binding.source === 'persisted') {
|
||||
return result('unverifiable')
|
||||
}
|
||||
const recognized = recognizeAgentProcess(evidence.foregroundProcess)
|
||||
if (recognized) {
|
||||
return result(
|
||||
isTuiAgent(entry.agentType) && recognized.agent !== entry.agentType ? 'exited' : 'live'
|
||||
)
|
||||
}
|
||||
return result(
|
||||
evidence.foregroundProcess !== null && evidence.hasChildProcesses === false
|
||||
? 'exited'
|
||||
: 'unverifiable'
|
||||
]
|
||||
)
|
||||
})
|
||||
)
|
||||
return new Map(resolved)
|
||||
}
|
||||
|
||||
private getRestoredAgentPtyBinding(entry: AgentStatusIpcPayload): RestoredAgentPtyBinding | null {
|
||||
const persisted = this.getPersistedRestoredAgentPtyBinding(entry)
|
||||
const current = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (persisted && current) {
|
||||
return current.ptyId === persisted.ptyId && current.incarnationId === persisted.incarnationId
|
||||
? {
|
||||
ptyId: current.ptyId,
|
||||
incarnationId: current.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
|
||||
source: 'current'
|
||||
}
|
||||
: persisted
|
||||
}
|
||||
if (current) {
|
||||
return {
|
||||
ptyId: current.ptyId,
|
||||
incarnationId: current.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
|
||||
source: 'current'
|
||||
}
|
||||
}
|
||||
return persisted
|
||||
}
|
||||
|
||||
private getPersistedRestoredAgentPtyBinding(
|
||||
entry: AgentStatusIpcPayload
|
||||
): RestoredAgentPtyBinding | null {
|
||||
if (!entry.worktreeId) {
|
||||
return null
|
||||
}
|
||||
const session = this.getWorkspaceSessionForWorktree(entry.worktreeId)
|
||||
for (const [ptyId, binding] of indexPersistedPtySurfaceBindings(session)) {
|
||||
if (
|
||||
binding.paneKey === entry.paneKey &&
|
||||
runtimeWorktreeIdsEqual(binding.worktreeId, entry.worktreeId)
|
||||
) {
|
||||
return {
|
||||
ptyId,
|
||||
incarnationId: binding.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(ptyId),
|
||||
source: 'persisted'
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private isRestoredAgentPtyBindingCurrent(
|
||||
entry: AgentStatusIpcPayload,
|
||||
binding: RestoredAgentPtyBinding
|
||||
): boolean {
|
||||
if (this.getPtyLifecycleGeneration(binding.ptyId) !== binding.lifecycleGeneration) {
|
||||
return false
|
||||
}
|
||||
const current = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (binding.source === 'current') {
|
||||
return current?.ptyId === binding.ptyId && current.incarnationId === binding.incarnationId
|
||||
}
|
||||
const persisted = this.getPersistedRestoredAgentPtyBinding(entry)
|
||||
return (
|
||||
current === null &&
|
||||
persisted?.ptyId === binding.ptyId &&
|
||||
persisted.incarnationId === binding.incarnationId
|
||||
)
|
||||
}
|
||||
|
||||
listRepos(): Repo[] {
|
||||
return this.store?.getRepos() ?? []
|
||||
}
|
||||
@@ -32525,6 +32579,7 @@ export class OrcaRuntimeService {
|
||||
if (state.incarnationId !== undefined) {
|
||||
if (pty.incarnationId && state.incarnationId && pty.incarnationId !== state.incarnationId) {
|
||||
this.invalidatePtyIncarnationHandle(ptyId)
|
||||
this.advancePtyLifecycleGeneration(ptyId)
|
||||
}
|
||||
pty.incarnationId = state.incarnationId
|
||||
}
|
||||
@@ -32556,6 +32611,9 @@ export class OrcaRuntimeService {
|
||||
pty.paneKey = state.paneKey
|
||||
}
|
||||
if (state.connected !== undefined) {
|
||||
if (state.connected && !pty.connected) {
|
||||
this.ptyProcessLiveness.invalidate(ptyId)
|
||||
}
|
||||
pty.connected = state.connected
|
||||
pty.disconnectedAt = state.connected ? null : (pty.disconnectedAt ?? Date.now())
|
||||
}
|
||||
@@ -38813,6 +38871,7 @@ export function resolveWorktreeScanCacheTtlMs(repo: Pick<Repo, 'path' | 'connect
|
||||
: WORKTREE_SCAN_CACHE_TTL_MS
|
||||
}
|
||||
const PTY_CONTROLLER_LIST_TIMEOUT_MS = 3000
|
||||
const PTY_PROCESS_INSPECTION_TIMEOUT_MS = 1_000
|
||||
// Why: the slice of the list budget reserved for the aggregate to collect the providers
|
||||
// that answered after a stalled one gives up.
|
||||
const PTY_CONTROLLER_LIST_PROVIDER_MARGIN_MS = 500
|
||||
@@ -40635,6 +40694,26 @@ function inferWorktreeIdFromPtyId(ptyId: string): string | null {
|
||||
return parsePtySessionId(ptyId).worktreeId
|
||||
}
|
||||
|
||||
function restoredAgentHookIdentity(entry: AgentStatusIpcPayload): string {
|
||||
return JSON.stringify([
|
||||
entry.paneKey,
|
||||
entry.receivedAt,
|
||||
entry.stateStartedAt,
|
||||
entry.state,
|
||||
entry.agentType,
|
||||
entry.connectionId,
|
||||
entry.worktreeId,
|
||||
entry.tabId,
|
||||
entry.terminalHandle,
|
||||
entry.launchToken,
|
||||
entry.prompt,
|
||||
entry.lastAssistantMessage,
|
||||
entry.toolName,
|
||||
entry.toolInput,
|
||||
entry.interrupted
|
||||
])
|
||||
}
|
||||
|
||||
function indexPersistedPtyWorktreeBindings(
|
||||
session: WorkspaceSessionState | null | undefined
|
||||
): ReadonlyMap<string, string> {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { PtyProcessLivenessBroker } from './pty-process-liveness-broker'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('PtyProcessLivenessBroker', () => {
|
||||
it('bounds a hanging probe and does not fan out another unresolved inspection', async () => {
|
||||
vi.useFakeTimers()
|
||||
let resolveInspection!: (value: {
|
||||
foregroundProcess: string | null
|
||||
hasChildProcesses: boolean
|
||||
}) => void
|
||||
const inspection = new Promise<{
|
||||
foregroundProcess: string | null
|
||||
hasChildProcesses: boolean
|
||||
}>((resolve) => {
|
||||
resolveInspection = resolve
|
||||
})
|
||||
const inspectProcess = vi.fn(() => inspection)
|
||||
const source = {
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess
|
||||
}
|
||||
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 })
|
||||
|
||||
const first = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
await expect(first).resolves.toEqual({
|
||||
status: 'unverifiable',
|
||||
reason: 'process inspection timed out'
|
||||
})
|
||||
await expect(
|
||||
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
).resolves.toEqual({ status: 'unverifiable', reason: 'process inspection timed out' })
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
|
||||
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
|
||||
await vi.runAllTimersAsync()
|
||||
await expect(
|
||||
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
).resolves.toEqual({
|
||||
status: 'live',
|
||||
foregroundProcess: 'codex',
|
||||
hasChildProcesses: true
|
||||
})
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('caches live evidence across sequential clients', async () => {
|
||||
const inspectProcess = vi.fn(async () => ({
|
||||
foregroundProcess: 'codex',
|
||||
hasChildProcesses: true
|
||||
}))
|
||||
const source = {
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess
|
||||
}
|
||||
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 })
|
||||
|
||||
await broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
await broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('treats only exact terminal_gone evidence as exited', async () => {
|
||||
const gone = vi.fn(async () => {
|
||||
throw new Error('terminal_gone')
|
||||
})
|
||||
const unavailable = vi.fn(async () => {
|
||||
throw new Error('socket_closed')
|
||||
})
|
||||
const onInspectionError = vi.fn()
|
||||
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100, onInspectionError })
|
||||
|
||||
await expect(
|
||||
broker.inspect({
|
||||
source: { getForegroundProcess: vi.fn(async () => null), inspectProcess: gone },
|
||||
ptyId: 'pty-gone',
|
||||
identity: 'incarnation-a'
|
||||
})
|
||||
).resolves.toEqual({ status: 'exited' })
|
||||
await expect(
|
||||
broker.inspect({
|
||||
source: { getForegroundProcess: vi.fn(async () => null), inspectProcess: unavailable },
|
||||
ptyId: 'pty-unavailable',
|
||||
identity: 'incarnation-b'
|
||||
})
|
||||
).resolves.toEqual({ status: 'unverifiable', reason: 'socket_closed' })
|
||||
expect(onInspectionError).toHaveBeenCalledExactlyOnceWith(
|
||||
'pty-unavailable',
|
||||
expect.objectContaining({ message: 'socket_closed' })
|
||||
)
|
||||
})
|
||||
|
||||
it('does not let an old incarnation populate the replacement cache entry', async () => {
|
||||
let resolveOld!: (value: { foregroundProcess: string; hasChildProcesses: boolean }) => void
|
||||
const oldInspection = new Promise<{
|
||||
foregroundProcess: string
|
||||
hasChildProcesses: boolean
|
||||
}>((resolve) => {
|
||||
resolveOld = resolve
|
||||
})
|
||||
const inspectProcess = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce(oldInspection)
|
||||
.mockResolvedValueOnce({ foregroundProcess: 'zsh', hasChildProcesses: false })
|
||||
const source = {
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess
|
||||
}
|
||||
const broker = new PtyProcessLivenessBroker({ timeoutMs: 1_000 })
|
||||
|
||||
const oldRead = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
const replacementRead = broker.inspect({
|
||||
source,
|
||||
ptyId: 'pty-1',
|
||||
identity: 'incarnation-b'
|
||||
})
|
||||
resolveOld({ foregroundProcess: 'codex', hasChildProcesses: true })
|
||||
|
||||
await expect(oldRead).resolves.toMatchObject({ status: 'live', foregroundProcess: 'codex' })
|
||||
await expect(replacementRead).resolves.toMatchObject({
|
||||
status: 'live',
|
||||
foregroundProcess: 'zsh'
|
||||
})
|
||||
await expect(
|
||||
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-b' })
|
||||
).resolves.toMatchObject({ status: 'live', foregroundProcess: 'zsh' })
|
||||
expect(inspectProcess).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,250 @@
|
||||
export type PtyProcessInspectionSource = {
|
||||
getForegroundProcess(ptyId: string): Promise<string | null>
|
||||
inspectProcess?: (ptyId: string) => Promise<{
|
||||
foregroundProcess: string | null
|
||||
hasChildProcesses: boolean
|
||||
unavailable?: true
|
||||
}>
|
||||
hasChildProcesses?: (ptyId: string) => Promise<boolean>
|
||||
}
|
||||
|
||||
export type PtyProcessLivenessEvidence =
|
||||
| {
|
||||
status: 'live'
|
||||
foregroundProcess: string | null
|
||||
hasChildProcesses: boolean | null
|
||||
}
|
||||
| { status: 'unverifiable'; reason: string }
|
||||
| { status: 'exited' }
|
||||
|
||||
type PtyProcessEvidenceEntry = {
|
||||
source: PtyProcessInspectionSource
|
||||
identity: string
|
||||
freshness: number
|
||||
failureCount: number
|
||||
evidence: PtyProcessLivenessEvidence | null
|
||||
expiresAt: number
|
||||
timedOut: boolean
|
||||
probe: Promise<PtyProcessLivenessEvidence> | null
|
||||
}
|
||||
|
||||
export type PtyProcessLivenessBrokerOptions = {
|
||||
timeoutMs: number
|
||||
liveTtlMs?: number
|
||||
unavailableBackoffBaseMs?: number
|
||||
unavailableBackoffMaxMs?: number
|
||||
now?: () => number
|
||||
onInspectionError?: (ptyId: string, error: unknown) => void
|
||||
}
|
||||
|
||||
const DEFAULT_LIVE_TTL_MS = 10_000
|
||||
const DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS = 3_000
|
||||
const DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS = 30_000
|
||||
|
||||
export class PtyProcessLivenessBroker {
|
||||
private readonly entries = new Map<string, PtyProcessEvidenceEntry>()
|
||||
private readonly now: () => number
|
||||
|
||||
constructor(private readonly options: PtyProcessLivenessBrokerOptions) {
|
||||
this.now = options.now ?? Date.now
|
||||
}
|
||||
|
||||
inspect(args: {
|
||||
source: PtyProcessInspectionSource
|
||||
ptyId: string
|
||||
identity: string
|
||||
freshness?: number
|
||||
reuseSettled?: boolean
|
||||
deadline?: number
|
||||
}): Promise<PtyProcessLivenessEvidence> {
|
||||
const freshness = args.freshness ?? 0
|
||||
const waitMs = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
this.options.timeoutMs,
|
||||
args.deadline === undefined ? this.options.timeoutMs : args.deadline - this.now()
|
||||
)
|
||||
)
|
||||
const existing = this.entries.get(args.ptyId)
|
||||
if (
|
||||
existing?.source === args.source &&
|
||||
existing.identity === args.identity &&
|
||||
existing.probe &&
|
||||
existing.freshness < freshness
|
||||
) {
|
||||
if (waitMs === 0) {
|
||||
return Promise.resolve({
|
||||
status: 'unverifiable',
|
||||
reason: 'process inspection timed out'
|
||||
})
|
||||
}
|
||||
return this.waitForProbe(args.ptyId, existing, waitMs).then(() =>
|
||||
existing.probe
|
||||
? { status: 'unverifiable', reason: 'process inspection timed out' }
|
||||
: this.inspect(args)
|
||||
)
|
||||
}
|
||||
if (
|
||||
existing?.source === args.source &&
|
||||
existing.identity === args.identity &&
|
||||
existing.freshness >= freshness
|
||||
) {
|
||||
if (existing.probe) {
|
||||
return existing.timedOut
|
||||
? Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
|
||||
: waitMs === 0
|
||||
? Promise.resolve({
|
||||
status: 'unverifiable',
|
||||
reason: 'process inspection timed out'
|
||||
})
|
||||
: this.waitForProbe(args.ptyId, existing, waitMs)
|
||||
}
|
||||
if (args.reuseSettled !== false && existing.evidence && existing.expiresAt > this.now()) {
|
||||
return Promise.resolve(existing.evidence)
|
||||
}
|
||||
}
|
||||
if (waitMs === 0) {
|
||||
return Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
|
||||
}
|
||||
|
||||
const failureCount =
|
||||
existing?.source === args.source && existing.identity === args.identity
|
||||
? existing.failureCount
|
||||
: 0
|
||||
const entry: PtyProcessEvidenceEntry = {
|
||||
source: args.source,
|
||||
identity: args.identity,
|
||||
freshness,
|
||||
failureCount,
|
||||
evidence: null,
|
||||
expiresAt: 0,
|
||||
timedOut: false,
|
||||
probe: null
|
||||
}
|
||||
const probe = this.runProbe(args.source, args.ptyId)
|
||||
.then((evidence) => {
|
||||
if (this.entries.get(args.ptyId) !== entry) {
|
||||
return evidence
|
||||
}
|
||||
entry.probe = null
|
||||
entry.timedOut = false
|
||||
entry.evidence = evidence
|
||||
if (evidence.status === 'unverifiable') {
|
||||
entry.failureCount += 1
|
||||
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
|
||||
} else if (evidence.status === 'live') {
|
||||
entry.failureCount = 0
|
||||
entry.expiresAt = this.now() + (this.options.liveTtlMs ?? DEFAULT_LIVE_TTL_MS)
|
||||
} else {
|
||||
entry.failureCount = 0
|
||||
entry.expiresAt = Number.POSITIVE_INFINITY
|
||||
}
|
||||
return evidence
|
||||
})
|
||||
.catch((error): PtyProcessLivenessEvidence => {
|
||||
if (this.entries.get(args.ptyId) === entry) {
|
||||
entry.probe = null
|
||||
entry.timedOut = false
|
||||
entry.failureCount += 1
|
||||
entry.evidence = { status: 'unverifiable', reason: describeError(error) }
|
||||
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
|
||||
}
|
||||
this.options.onInspectionError?.(args.ptyId, error)
|
||||
return { status: 'unverifiable', reason: describeError(error) }
|
||||
})
|
||||
entry.probe = probe
|
||||
this.entries.set(args.ptyId, entry)
|
||||
return this.waitForProbe(args.ptyId, entry, waitMs)
|
||||
}
|
||||
|
||||
invalidate(ptyId: string): void {
|
||||
this.entries.delete(ptyId)
|
||||
}
|
||||
|
||||
invalidateAll(): void {
|
||||
this.entries.clear()
|
||||
}
|
||||
|
||||
getPendingCount(): number {
|
||||
let count = 0
|
||||
for (const entry of this.entries.values()) {
|
||||
if (entry.probe) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
private async runProbe(
|
||||
source: PtyProcessInspectionSource,
|
||||
ptyId: string
|
||||
): Promise<PtyProcessLivenessEvidence> {
|
||||
try {
|
||||
if (source.inspectProcess) {
|
||||
const inspection = await source.inspectProcess(ptyId)
|
||||
return inspection.unavailable
|
||||
? { status: 'unverifiable', reason: 'process inspection unavailable' }
|
||||
: {
|
||||
status: 'live',
|
||||
foregroundProcess: inspection.foregroundProcess,
|
||||
hasChildProcesses: inspection.hasChildProcesses
|
||||
}
|
||||
}
|
||||
const foregroundProcess = await source.getForegroundProcess(ptyId)
|
||||
return {
|
||||
status: 'live',
|
||||
foregroundProcess,
|
||||
hasChildProcesses: source.hasChildProcesses ? await source.hasChildProcesses(ptyId) : null
|
||||
}
|
||||
} catch (error) {
|
||||
if (isTerminalGoneError(error)) {
|
||||
return { status: 'exited' }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private waitForProbe(
|
||||
ptyId: string,
|
||||
entry: PtyProcessEvidenceEntry,
|
||||
timeoutMs: number
|
||||
): Promise<PtyProcessLivenessEvidence> {
|
||||
const probe = entry.probe
|
||||
if (!probe) {
|
||||
return Promise.resolve(
|
||||
entry.evidence ?? { status: 'unverifiable', reason: 'process inspection unavailable' }
|
||||
)
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
if (this.entries.get(ptyId) === entry && entry.probe === probe) {
|
||||
entry.timedOut = true
|
||||
}
|
||||
resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
|
||||
}, timeoutMs)
|
||||
void probe.then((evidence) => {
|
||||
clearTimeout(timeout)
|
||||
resolve(evidence)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
private unavailableBackoffMs(failureCount: number): number {
|
||||
const base = this.options.unavailableBackoffBaseMs ?? DEFAULT_UNAVAILABLE_BACKOFF_BASE_MS
|
||||
const max = this.options.unavailableBackoffMaxMs ?? DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS
|
||||
return Math.min(max, base * 2 ** Math.max(0, failureCount - 1))
|
||||
}
|
||||
}
|
||||
|
||||
function isTerminalGoneError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? String((error as { code?: unknown }).code)
|
||||
: ''
|
||||
return message === 'terminal_gone' || code === 'terminal_gone'
|
||||
}
|
||||
|
||||
function describeError(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
Reference in New Issue
Block a user