mirror of
https://github.com/stablyai/orca.git
synced 2026-10-24 08:03:58 +00:00
fix(runtime): bind restored agents to process authority
This commit is contained in:
Jinwoo-H
@@ -10293,7 +10293,7 @@
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts --fileParallelism=false --maxWorkers=1 src/main/runtime/restored-agent-authority-resolver.test.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts",
|
||||
"cd mobile && pnpm exec vitest run src/worktree/agent-row-display.test.ts src/transport/stable-logical-rpc-client.test.ts",
|
||||
"pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1",
|
||||
"pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1",
|
||||
@@ -10305,6 +10305,7 @@
|
||||
"src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts",
|
||||
"src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts",
|
||||
"src/main/runtime/orca-runtime.test.ts",
|
||||
"src/main/runtime/restored-agent-authority-resolver.test.ts",
|
||||
"src/main/runtime/pty-process-liveness-broker.test.ts",
|
||||
"src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts",
|
||||
"src/cli/handlers/orchestration.test.ts",
|
||||
@@ -10322,7 +10323,8 @@
|
||||
"src/main/ssh/ssh-remote-orca-cli.test.ts",
|
||||
"tests/e2e/orchestration-worker-terminal-visibility.spec.ts",
|
||||
"tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts",
|
||||
"tests/e2e/orchestration-worker-settlement-release-cli.spec.ts"
|
||||
"tests/e2e/orchestration-worker-settlement-release-cli.spec.ts",
|
||||
"tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
@@ -10349,6 +10351,15 @@
|
||||
"ambiguous, incomplete, or mismatched legacy identities remain fenced and deferred"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/restored-agent-authority-resolver.test.ts",
|
||||
"assertions": [
|
||||
"one restored hook cannot borrow a replacement PTY incarnation or another execution host",
|
||||
"conflicting current and persisted bindings remain unverifiable",
|
||||
"incarnation identity survives local lifecycle-generation changes",
|
||||
"retired hook commitments are released without unbounded growth"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/orca-runtime.test.ts",
|
||||
"assertions": [
|
||||
@@ -10359,7 +10370,10 @@
|
||||
"a connected live PTY remains inventoried when its user-visible tab is missing",
|
||||
"legacy clients retain the exact restored-row suppression",
|
||||
"inspection timeout, transport loss, missing bindings, and stale incarnations remain unverifiable",
|
||||
"a handle-qualified cold binding remains visible when owning inventory loses contact",
|
||||
"terminal_gone is the only inspection rejection that retires the exact restored row",
|
||||
"fresh exact inventory cannot be overwritten by older cached terminal_gone evidence after later contact loss",
|
||||
"slow foreground inspection cannot manufacture agent exit or certify a replacement incarnation",
|
||||
"sequential inventory clients reuse bounded process evidence instead of polling each restored row"
|
||||
]
|
||||
},
|
||||
@@ -10508,10 +10522,10 @@
|
||||
"date": "2026-08-23",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts --fileParallelism=false --maxWorkers=1 src/main/runtime/restored-agent-authority-resolver.test.ts src/main/runtime/pty-process-liveness-broker.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/worktree-catalog-snapshot-method.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/remote-runtime-client.test.ts src/shared/remote-runtime-request-connection.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/cli/runtime/websocket-transport.test.ts tests/e2e/cross-version-wire/cross-version-terminal-wire.unit.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 15.33,
|
||||
"summary": "Eight focused runtime and wire files passed 1,273 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, cold persisted binding, incarnation and hook replacement races, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, and encrypted desktop/CLI capability advertisement. Mobile typecheck and 25 focused mobile tests passed separately."
|
||||
"durationSeconds": 49.44,
|
||||
"summary": "Ten focused runtime and wire files passed 1,292 tests with one skip, including bounded inspection timeout, exact terminal_gone absence, three-poll fresh-presence reconciliation, cold handle-qualified contact loss, incarnation/host/hook replacement fencing, slow foreground completion, callback isolation, sequential-client call counts, live/unverifiable/exited/hidden-tab/legacy projection, encrypted desktop/CLI capability advertisement, and cross-version terminal wire compatibility. Full mobile typecheck and 3,674 tests passed separately."
|
||||
},
|
||||
{
|
||||
"date": "2026-08-13",
|
||||
|
||||
@@ -9892,6 +9892,45 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(getForegroundProcess).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('does not manufacture an agent exit when foreground inspection is slow', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { runtime, batches } = createSideEffectRuntime()
|
||||
syncSinglePty(runtime)
|
||||
let resolveProcess!: (process: string) => void
|
||||
const getForegroundProcess = vi.fn(
|
||||
() =>
|
||||
new Promise<string>((resolve) => {
|
||||
resolveProcess = resolve
|
||||
})
|
||||
)
|
||||
runtime.setPtyController({
|
||||
write: () => true,
|
||||
kill: () => true,
|
||||
getForegroundProcess
|
||||
})
|
||||
runtime.ingestSyntheticTitleFrame('pty-1', '\x1b]0;Codex ready\x07')
|
||||
batches.length = 0
|
||||
|
||||
runtime.onPtyData('pty-1', '\x1b]0;bichir\x07', 101)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
|
||||
expect(batches.flatMap((batch) => batch.facts)).not.toContainEqual({
|
||||
kind: 'agent-exited'
|
||||
})
|
||||
runtime.onPtySpawned('pty-1', 'replacement-incarnation', {
|
||||
awaitsRegistration: false
|
||||
})
|
||||
resolveProcess('zsh')
|
||||
await vi.runAllTimersAsync()
|
||||
expect(batches.flatMap((batch) => batch.facts)).not.toContainEqual({
|
||||
kind: 'agent-exited'
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not confirm an agent exit from a foreground read predating its title', async () => {
|
||||
const { runtime, batches } = createSideEffectRuntime()
|
||||
syncSinglePty(runtime)
|
||||
@@ -9921,7 +9960,7 @@ describe('OrcaRuntimeService', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('treats synchronous foreground read failures as unavailable', async () => {
|
||||
it('does not turn synchronous foreground read failures into agent exits', async () => {
|
||||
const { runtime, batches } = createSideEffectRuntime()
|
||||
syncSinglePty(runtime)
|
||||
runtime.ingestSyntheticTitleFrame('pty-1', '\x1b]0;Codex ready\x07')
|
||||
@@ -9936,9 +9975,9 @@ describe('OrcaRuntimeService', () => {
|
||||
|
||||
runtime.onPtyData('pty-1', '\x1b]0;bichir\x07', 100)
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(batches.flatMap((batch) => batch.facts)).toContainEqual({ kind: 'agent-exited' })
|
||||
)
|
||||
await vi.waitFor(() => expect(getForegroundProcess).toHaveBeenCalledOnce())
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
expect(batches.flatMap((batch) => batch.facts)).not.toContainEqual({ kind: 'agent-exited' })
|
||||
expect(getForegroundProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
@@ -37846,6 +37885,58 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('persisted-pty')
|
||||
})
|
||||
|
||||
it('retains a handle-qualified cold binding when owning inventory loses contact', async () => {
|
||||
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
|
||||
const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
const session = makeWorkspaceSessionWithHeadlessTerminal({
|
||||
terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId }
|
||||
})
|
||||
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const now = Date.now()
|
||||
const inspectProcess = vi.fn(async () => ({
|
||||
foregroundProcess: null,
|
||||
hasChildProcesses: false,
|
||||
unavailable: true as const
|
||||
}))
|
||||
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'host-tab',
|
||||
terminalHandle: 'term_restored',
|
||||
state: 'working',
|
||||
prompt: 'await reconnect',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess,
|
||||
listProcesses: vi.fn(async () => {
|
||||
throw new Error('socket_closed')
|
||||
})
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
expect(inspectProcess).toHaveBeenCalledExactlyOnceWith('persisted-pty')
|
||||
})
|
||||
|
||||
it('keeps conflicting persisted and current PTY bindings unverifiable', async () => {
|
||||
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
|
||||
const session = makeWorkspaceSessionWithHeadlessTerminal({
|
||||
@@ -37941,6 +38032,55 @@ describe('OrcaRuntimeService', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('does not use a replacement incarnation to contradict exact terminal absence', async () => {
|
||||
const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID)
|
||||
const originalIncarnation = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
const session = makeWorkspaceSessionWithHeadlessTerminal({
|
||||
terminalPtyIncarnationsByPaneKey: { [paneKey]: originalIncarnation }
|
||||
})
|
||||
const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const now = Date.now()
|
||||
const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'host-tab',
|
||||
state: 'working',
|
||||
prompt: 'old incarnation completed',
|
||||
agentType: 'codex',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess: vi.fn(async () => {
|
||||
throw new Error('terminal_gone')
|
||||
}),
|
||||
listProcesses: vi.fn(async () => [
|
||||
{
|
||||
id: 'persisted-pty',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
title: 'Codex',
|
||||
incarnationId: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb',
|
||||
terminalHandle: 'term_replacement'
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual(
|
||||
[]
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps cached terminal_gone unverifiable when owning inventory re-observes the PTY', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
@@ -37952,7 +38092,7 @@ describe('OrcaRuntimeService', () => {
|
||||
const listProcesses = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValue([
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
id: 'pty-1',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
@@ -37961,6 +38101,7 @@ describe('OrcaRuntimeService', () => {
|
||||
terminalHandle: 'term_restored'
|
||||
}
|
||||
])
|
||||
.mockRejectedValueOnce(new Error('socket_closed'))
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
@@ -38013,6 +38154,7 @@ describe('OrcaRuntimeService', () => {
|
||||
|
||||
const first = await runtime.getWorktreePs(undefined, true, true)
|
||||
const second = await runtime.getWorktreePs(undefined, true, true)
|
||||
const third = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(
|
||||
first.worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents
|
||||
@@ -38026,6 +38168,15 @@ describe('OrcaRuntimeService', () => {
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
expect(
|
||||
third.worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
@@ -38179,6 +38330,76 @@ describe('OrcaRuntimeService', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('does not borrow local PTY evidence for a restored SSH hook', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
const now = Date.now()
|
||||
const inspectProcess = vi.fn(async () => ({
|
||||
foregroundProcess: 'codex',
|
||||
hasChildProcesses: true
|
||||
}))
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'tab-1',
|
||||
state: 'working',
|
||||
prompt: 'remote hook',
|
||||
agentType: 'codex',
|
||||
connectionId: 'ssh-1',
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100,
|
||||
restoredUnconfirmed: true
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: vi.fn(async () => 'codex'),
|
||||
inspectProcess,
|
||||
listProcesses: vi.fn(async () => [])
|
||||
})
|
||||
runtime.registerPty('pty-1', TEST_WORKTREE_ID, null, {
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
incarnationId: 'incarnation-a'
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Codex',
|
||||
activeLeafId: leafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-1',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: 'pty-1'
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
const { worktrees } = await runtime.getWorktreePs(undefined, true, true)
|
||||
|
||||
expect(worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)?.agents).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
expect(inspectProcess).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('fences a restored-agent inspection across PTY reincarnation', async () => {
|
||||
const leafId = '33333333-3333-4333-8333-333333333333'
|
||||
const paneKey = `tab-1:${leafId}`
|
||||
@@ -38256,6 +38477,18 @@ describe('OrcaRuntimeService', () => {
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
const afterReplacement = await runtime.getWorktreePs(undefined, true, true)
|
||||
expect(
|
||||
afterReplacement.worktrees.find((worktree) => worktree.worktreeId === TEST_WORKTREE_ID)
|
||||
?.agents
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
paneKey,
|
||||
restoredUnconfirmed: true,
|
||||
agentLiveness: 'unverifiable'
|
||||
})
|
||||
])
|
||||
expect(inspectProcess).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('does not publish a restored hook snapshot replaced during inspection', async () => {
|
||||
|
||||
@@ -661,6 +661,12 @@ import { createRuntimeBrowserCommands } from './runtime-browser-commands-factory
|
||||
import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency'
|
||||
import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity'
|
||||
import { PtyProcessLivenessBroker } from './pty-process-liveness-broker'
|
||||
import {
|
||||
RestoredAgentAuthorityResolver,
|
||||
sameRestoredAgentProcessAuthority,
|
||||
type RestoredAgentAuthorityBinding,
|
||||
type RestoredAgentAuthorityHook
|
||||
} from './restored-agent-authority-resolver'
|
||||
import {
|
||||
buildHeadlessTerminalSplitLayout,
|
||||
countTerminalLayoutLeaves,
|
||||
@@ -1628,6 +1634,8 @@ type PtyForegroundAgentRefresh = {
|
||||
|
||||
type PtyForegroundProcessRead = {
|
||||
controller: RuntimePtyController
|
||||
identity: string
|
||||
status: RestoredAgentLiveness
|
||||
process: string | null
|
||||
available: boolean
|
||||
}
|
||||
@@ -1640,13 +1648,6 @@ type RestoredAgentLivenessResolution = {
|
||||
hasExactBinding: boolean
|
||||
}
|
||||
|
||||
type RestoredAgentPtyBinding = {
|
||||
ptyId: string
|
||||
incarnationId: PtyIncarnationId | null
|
||||
lifecycleGeneration: number
|
||||
source: 'current' | 'persisted'
|
||||
}
|
||||
|
||||
function copySleepingAgentLaunchConfig(
|
||||
config: SleepingAgentLaunchConfig
|
||||
): SleepingAgentLaunchConfig {
|
||||
@@ -1992,6 +1993,7 @@ type PtyControllerInventory = Readonly<{
|
||||
// Why: livePtyIds is worktree-scoped when a target is given; absence proofs
|
||||
// must consult the unscoped inventory or a misattributed live PTY reads as dead.
|
||||
allLivePtyIds: ReadonlySet<string>
|
||||
incarnationIdByPtyId: ReadonlyMap<string, string>
|
||||
terminalIdentityByPtyId: ReadonlyMap<string, PtyControllerTerminalIdentity>
|
||||
queriedHostIds: ReadonlySet<ExecutionHostId>
|
||||
}>
|
||||
@@ -3186,6 +3188,7 @@ export class OrcaRuntimeService {
|
||||
console.warn(`[runtime] PTY process inspection failed for ${ptyId}`, error)
|
||||
}
|
||||
})
|
||||
private readonly restoredAgentAuthority = new RestoredAgentAuthorityResolver()
|
||||
private ptyDelayedForegroundSnapshotTitleObservations = new Map<string, number>()
|
||||
// Why a set and not a timer: the intent is retired by the exit it explains, or
|
||||
// by the next lifecycle generation on that id (advancePtyLifecycleGeneration),
|
||||
@@ -19061,7 +19064,7 @@ export class OrcaRuntimeService {
|
||||
private readPtyForegroundProcessFromController(
|
||||
ptyId: string,
|
||||
afterTitleObservation = 0,
|
||||
options: { reuseSettled?: boolean } = {}
|
||||
options: { reuseSettled?: boolean; waitForSettlement?: boolean } = {}
|
||||
): Promise<PtyForegroundProcessRead> | null {
|
||||
const controller = this.ptyController
|
||||
if (!controller) {
|
||||
@@ -19074,10 +19077,13 @@ export class OrcaRuntimeService {
|
||||
ptyId,
|
||||
identity,
|
||||
freshness: afterTitleObservation,
|
||||
reuseSettled: options.reuseSettled
|
||||
reuseSettled: options.reuseSettled,
|
||||
waitForSettlement: options.waitForSettlement
|
||||
})
|
||||
.then((evidence) => ({
|
||||
controller,
|
||||
identity,
|
||||
status: evidence.status,
|
||||
process: evidence.status === 'live' ? evidence.foregroundProcess : null,
|
||||
available: evidence.status === 'live'
|
||||
}))
|
||||
@@ -19096,15 +19102,18 @@ export class OrcaRuntimeService {
|
||||
const foregroundRead = this.readPtyForegroundProcessFromController(
|
||||
ptyId,
|
||||
titleObservedAt ?? 0,
|
||||
{ reuseSettled: false }
|
||||
{ reuseSettled: false, waitForSettlement: true }
|
||||
)
|
||||
if (!pty?.connected || !foregroundRead) {
|
||||
this.recordTerminalSideEffectFact(ptyId, { kind: 'agent-exited' })
|
||||
return
|
||||
}
|
||||
void foregroundRead.then((result) => {
|
||||
const current = this.ptysById.get(ptyId)
|
||||
if (current !== pty || !current.connected) {
|
||||
if (
|
||||
current !== pty ||
|
||||
!current.connected ||
|
||||
result.identity !== this.getPtyProcessEvidenceIdentity(ptyId)
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (current.lastOscTitleAt !== titleObservedAt && current.lastAgentStatus !== null) {
|
||||
@@ -19133,6 +19142,9 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
return
|
||||
}
|
||||
if (result.status === 'unverifiable') {
|
||||
return
|
||||
}
|
||||
this.recordTerminalSideEffectFact(ptyId, { kind: 'agent-exited' })
|
||||
})
|
||||
}
|
||||
@@ -19249,7 +19261,11 @@ export class OrcaRuntimeService {
|
||||
return false
|
||||
}
|
||||
const result = await foregroundRead
|
||||
if (result.controller !== this.ptyController || !result.available) {
|
||||
if (
|
||||
result.controller !== this.ptyController ||
|
||||
result.identity !== this.getPtyProcessEvidenceIdentity(ptyId) ||
|
||||
!result.available
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const foregroundProcess = result.process
|
||||
@@ -19980,7 +19996,9 @@ export class OrcaRuntimeService {
|
||||
const ptyProcessEvidenceDeadline = Date.now() + PTY_CONTROLLER_LIST_TIMEOUT_MS
|
||||
// Why: worktree.ps backs the mobile sidebar, so it must use the same
|
||||
// host-owned imported-worktree visibility gate as worktree.list/desktop.
|
||||
const freshPtyLiveness = await this.refreshPtyWorktreeRecordsFromController(resolvedWorktrees)
|
||||
const freshPtyInventory =
|
||||
await this.refreshPtyWorktreeRecordsWithControllerInventory(resolvedWorktrees)
|
||||
const freshPtyLiveness = freshPtyInventory?.livePtyIds ?? null
|
||||
const repoById = new Map((this.store?.getRepos() ?? []).map((repo) => [repo.id, repo]))
|
||||
const platformByRepoId = resolvedWorktreeSnapshot.platformByRepoId
|
||||
const summaries = new Map<string, RuntimeWorktreePsSummary>()
|
||||
@@ -20337,7 +20355,7 @@ export class OrcaRuntimeService {
|
||||
missingRuntimeWorktreeIds,
|
||||
mirroredWorktreeIdByTabId,
|
||||
connectedPtyEvidence,
|
||||
freshPtyLiveness,
|
||||
freshPtyInventory,
|
||||
restoredAgentPresenceSupported,
|
||||
ptyProcessEvidenceDeadline
|
||||
)
|
||||
@@ -20364,7 +20382,7 @@ export class OrcaRuntimeService {
|
||||
paneKeys: ReadonlySet<string>
|
||||
ptyIds: ReadonlySet<string>
|
||||
},
|
||||
freshPtyLiveness: ReadonlySet<string> | null,
|
||||
freshPtyInventory: PtyControllerInventory | null,
|
||||
restoredAgentPresenceSupported: boolean,
|
||||
ptyProcessEvidenceDeadline: number
|
||||
): Promise<void> {
|
||||
@@ -20417,7 +20435,7 @@ export class OrcaRuntimeService {
|
||||
? await this.resolveRestoredAgentLiveness(
|
||||
hookEntriesAtInspection,
|
||||
ptyProcessEvidenceDeadline,
|
||||
freshPtyLiveness
|
||||
freshPtyInventory
|
||||
)
|
||||
: new Map<string, RestoredAgentLivenessResolution>()
|
||||
const hookEntries = restoredAgentPresenceSupported
|
||||
@@ -20570,33 +20588,28 @@ export class OrcaRuntimeService {
|
||||
private async resolveRestoredAgentLiveness(
|
||||
entries: readonly AgentStatusIpcPayload[],
|
||||
deadline: number,
|
||||
freshPtyLiveness: ReadonlySet<string> | null
|
||||
freshPtyInventory: PtyControllerInventory | null
|
||||
): Promise<Map<string, RestoredAgentLivenessResolution>> {
|
||||
const restoredEntries = entries.filter((entry) => entry.restoredUnconfirmed === true)
|
||||
this.restoredAgentAuthority.retain(
|
||||
new Set(restoredEntries.map((entry) => restoredAgentHookIdentity(entry)))
|
||||
)
|
||||
const resolved = await Promise.all(
|
||||
restoredEntries.map(async (entry): Promise<[string, RestoredAgentLivenessResolution]> => {
|
||||
const hookIdentity = restoredAgentHookIdentity(entry)
|
||||
const binding = this.getRestoredAgentPtyBinding(entry)
|
||||
const authority = this.getRestoredAgentAuthority(entry, freshPtyInventory)
|
||||
const binding = authority.binding
|
||||
const result = (
|
||||
liveness: RestoredAgentLiveness,
|
||||
hasExactBinding = binding !== null
|
||||
hasExactBinding = authority.hasExactBinding
|
||||
): [string, RestoredAgentLivenessResolution] => [
|
||||
entry.paneKey,
|
||||
{ liveness, hookIdentity, hasExactBinding }
|
||||
]
|
||||
if (!binding) {
|
||||
return result('unverifiable', false)
|
||||
}
|
||||
const currentAtStart = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (
|
||||
currentAtStart &&
|
||||
(currentAtStart.ptyId !== binding.ptyId ||
|
||||
(binding.incarnationId !== null &&
|
||||
currentAtStart.incarnationId !== binding.incarnationId))
|
||||
) {
|
||||
return result('unverifiable')
|
||||
}
|
||||
if (currentAtStart && this.isPtyKnownExited(currentAtStart.ptyId)) {
|
||||
if (this.isPtyKnownExited(binding.ptyId)) {
|
||||
return result('exited')
|
||||
}
|
||||
const controller = this.ptyController
|
||||
@@ -20608,17 +20621,20 @@ export class OrcaRuntimeService {
|
||||
ptyId: binding.ptyId,
|
||||
identity:
|
||||
binding.incarnationId ?? `runtime:${this.runtimeId}:${binding.lifecycleGeneration}`,
|
||||
deadline
|
||||
deadline,
|
||||
owningInventoryObservedPty: this.inventoryProvesRestoredBindingPresent(
|
||||
freshPtyInventory,
|
||||
binding
|
||||
)
|
||||
})
|
||||
const currentAuthority = this.getRestoredAgentAuthority(entry, freshPtyInventory).binding
|
||||
if (
|
||||
controller !== this.ptyController ||
|
||||
!this.isRestoredAgentPtyBindingCurrent(entry, binding)
|
||||
!currentAuthority ||
|
||||
!sameRestoredAgentProcessAuthority(currentAuthority, binding)
|
||||
) {
|
||||
return result('unverifiable')
|
||||
}
|
||||
if (evidence.status === 'exited' && freshPtyLiveness?.has(binding.ptyId)) {
|
||||
return result('unverifiable')
|
||||
}
|
||||
if (evidence.status !== 'live') {
|
||||
return result(evidence.status)
|
||||
}
|
||||
@@ -20641,33 +20657,26 @@ export class OrcaRuntimeService {
|
||||
return new Map(resolved)
|
||||
}
|
||||
|
||||
private getRestoredAgentPtyBinding(entry: AgentStatusIpcPayload): RestoredAgentPtyBinding | null {
|
||||
const persisted = this.getPersistedRestoredAgentPtyBinding(entry)
|
||||
const current = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (persisted && current) {
|
||||
return current.ptyId === persisted.ptyId && current.incarnationId === persisted.incarnationId
|
||||
? {
|
||||
ptyId: current.ptyId,
|
||||
incarnationId: current.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
|
||||
source: 'current'
|
||||
}
|
||||
: persisted
|
||||
private getRestoredAgentAuthority(
|
||||
entry: AgentStatusIpcPayload,
|
||||
inventory: PtyControllerInventory | null
|
||||
) {
|
||||
const hook = this.getRestoredAgentAuthorityHook(entry)
|
||||
if (!hook) {
|
||||
return { binding: null, hasExactBinding: false }
|
||||
}
|
||||
if (current) {
|
||||
return {
|
||||
ptyId: current.ptyId,
|
||||
incarnationId: current.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
|
||||
source: 'current'
|
||||
}
|
||||
}
|
||||
return persisted
|
||||
return this.restoredAgentAuthority.resolve({
|
||||
hook,
|
||||
current: this.getCurrentRestoredAgentBinding(entry, hook, inventory),
|
||||
persisted: this.getPersistedRestoredAgentPtyBinding(entry, hook, inventory)
|
||||
})
|
||||
}
|
||||
|
||||
private getPersistedRestoredAgentPtyBinding(
|
||||
entry: AgentStatusIpcPayload
|
||||
): RestoredAgentPtyBinding | null {
|
||||
entry: AgentStatusIpcPayload,
|
||||
hook: RestoredAgentAuthorityHook,
|
||||
inventory: PtyControllerInventory | null
|
||||
): RestoredAgentAuthorityBinding | null {
|
||||
if (!entry.worktreeId) {
|
||||
return null
|
||||
}
|
||||
@@ -20677,34 +20686,156 @@ export class OrcaRuntimeService {
|
||||
binding.paneKey === entry.paneKey &&
|
||||
runtimeWorktreeIdsEqual(binding.worktreeId, entry.worktreeId)
|
||||
) {
|
||||
const controllerIdentity = inventory?.terminalIdentityByPtyId.get(ptyId)
|
||||
const encodedHost = getPtyExecutionHost(ptyId)
|
||||
if (
|
||||
encodedHost === 'foreign' ||
|
||||
(encodedHost !== null && encodedHost !== hook.hostKey) ||
|
||||
(hook.hostKey.startsWith('wsl:') &&
|
||||
controllerIdentity?.wslDistro?.toLowerCase() !== hook.hostKey.slice(4))
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
ptyId,
|
||||
incarnationId: binding.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(ptyId),
|
||||
source: 'persisted'
|
||||
source: 'persisted',
|
||||
paneKey: binding.paneKey,
|
||||
worktreeKey: hook.worktreeKey,
|
||||
hostKey: hook.hostKey,
|
||||
...((controllerIdentity?.handle ?? hook.terminalHandle)
|
||||
? { terminalHandle: controllerIdentity?.handle ?? hook.terminalHandle }
|
||||
: {})
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private isRestoredAgentPtyBindingCurrent(
|
||||
private getCurrentRestoredAgentBinding(
|
||||
entry: AgentStatusIpcPayload,
|
||||
binding: RestoredAgentPtyBinding
|
||||
): boolean {
|
||||
if (this.getPtyLifecycleGeneration(binding.ptyId) !== binding.lifecycleGeneration) {
|
||||
hook: RestoredAgentAuthorityHook,
|
||||
inventory: PtyControllerInventory | null
|
||||
): RestoredAgentAuthorityBinding | null {
|
||||
const current = this.getExactPtyRecordForPaneKey(entry.paneKey)
|
||||
if (
|
||||
!current ||
|
||||
!runtimeWorktreeIdsEqual(current.worktreeId, entry.worktreeId ?? '') ||
|
||||
!this.ptyMatchesRestoredAgentHost(current, hook.hostKey)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const controllerIdentity = inventory?.terminalIdentityByPtyId.get(current.ptyId)
|
||||
return {
|
||||
ptyId: current.ptyId,
|
||||
incarnationId: current.incarnationId,
|
||||
lifecycleGeneration: this.getPtyLifecycleGeneration(current.ptyId),
|
||||
source: 'current',
|
||||
paneKey: entry.paneKey,
|
||||
worktreeKey: hook.worktreeKey,
|
||||
hostKey: hook.hostKey,
|
||||
...((controllerIdentity?.handle ??
|
||||
this.handleByPtyId.get(current.ptyId) ??
|
||||
hook.terminalHandle)
|
||||
? {
|
||||
terminalHandle:
|
||||
controllerIdentity?.handle ??
|
||||
this.handleByPtyId.get(current.ptyId) ??
|
||||
hook.terminalHandle
|
||||
}
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
private getRestoredAgentAuthorityHook(
|
||||
entry: AgentStatusIpcPayload
|
||||
): RestoredAgentAuthorityHook | null {
|
||||
if (!entry.worktreeId) {
|
||||
return null
|
||||
}
|
||||
const worktreeHost = this.tryGetWorkspaceSessionHostIdForWorktree(entry.worktreeId)
|
||||
if (!worktreeHost) {
|
||||
return null
|
||||
}
|
||||
let hostKey: string
|
||||
if (isWslHookRelayConnectionId(entry.connectionId)) {
|
||||
if (worktreeHost !== LOCAL_EXECUTION_HOST_ID) {
|
||||
return null
|
||||
}
|
||||
hostKey = entry.connectionId!.toLowerCase()
|
||||
} else if (entry.connectionId) {
|
||||
hostKey = toSshExecutionHostId(entry.connectionId)
|
||||
if (hostKey !== worktreeHost) {
|
||||
return null
|
||||
}
|
||||
} else {
|
||||
hostKey = worktreeHost
|
||||
}
|
||||
return {
|
||||
identity: restoredAgentHookIdentity(entry),
|
||||
paneKey: entry.paneKey,
|
||||
worktreeKey: runtimeWorktreeIdentityKey(entry.worktreeId),
|
||||
hostKey,
|
||||
...(entry.terminalHandle ? { terminalHandle: entry.terminalHandle } : {})
|
||||
}
|
||||
}
|
||||
|
||||
private getExactPtyRecordForPaneKey(paneKey: string): RuntimePtyWorktreeRecord | null {
|
||||
const parsed = parsePaneKey(paneKey)
|
||||
const leaf = parsed ? this.leaves.get(this.getLeafKey(parsed.tabId, parsed.leafId)) : undefined
|
||||
const candidates = new Set<RuntimePtyWorktreeRecord>()
|
||||
if (leaf?.ptyId) {
|
||||
const pty = this.ptysById.get(leaf.ptyId)
|
||||
if (pty) {
|
||||
candidates.add(pty)
|
||||
}
|
||||
}
|
||||
for (const pty of this.ptysById.values()) {
|
||||
if (pty.paneKey === paneKey) {
|
||||
candidates.add(pty)
|
||||
}
|
||||
}
|
||||
const connected = [...candidates].filter((pty) => pty.connected)
|
||||
return connected.length === 1
|
||||
? connected[0]!
|
||||
: candidates.size === 1
|
||||
? [...candidates][0]!
|
||||
: null
|
||||
}
|
||||
|
||||
private ptyMatchesRestoredAgentHost(pty: RuntimePtyWorktreeRecord, hostKey: string): boolean {
|
||||
const encodedHost = getPtyExecutionHost(pty.ptyId)
|
||||
if (encodedHost === 'foreign') {
|
||||
return false
|
||||
}
|
||||
const current = this.getPtyRecordForPaneKey(entry.paneKey)
|
||||
if (binding.source === 'current') {
|
||||
return current?.ptyId === binding.ptyId && current.incarnationId === binding.incarnationId
|
||||
if (encodedHost !== null) {
|
||||
return encodedHost === hostKey
|
||||
}
|
||||
const persisted = this.getPersistedRestoredAgentPtyBinding(entry)
|
||||
return (
|
||||
current === null &&
|
||||
persisted?.ptyId === binding.ptyId &&
|
||||
persisted.incarnationId === binding.incarnationId
|
||||
)
|
||||
if (hostKey.startsWith('wsl:')) {
|
||||
return (
|
||||
pty.connectionId === null && pty.wslDistro?.toLowerCase() === hostKey.slice(4).toLowerCase()
|
||||
)
|
||||
}
|
||||
const parsedHost = parseExecutionHostId(hostKey)
|
||||
if (parsedHost?.kind === 'ssh') {
|
||||
return pty.connectionId === parsedHost.targetId
|
||||
}
|
||||
return pty.connectionId === null && !pty.wslDistro
|
||||
}
|
||||
|
||||
private inventoryProvesRestoredBindingPresent(
|
||||
inventory: PtyControllerInventory | null,
|
||||
binding: RestoredAgentAuthorityBinding
|
||||
): boolean {
|
||||
if (!inventory?.livePtyIds.has(binding.ptyId) || !binding.incarnationId) {
|
||||
return false
|
||||
}
|
||||
if (inventory.incarnationIdByPtyId.get(binding.ptyId) !== binding.incarnationId) {
|
||||
return false
|
||||
}
|
||||
const identity = inventory.terminalIdentityByPtyId.get(binding.ptyId)
|
||||
return !binding.terminalHandle || identity?.handle === binding.terminalHandle
|
||||
}
|
||||
|
||||
listRepos(): Repo[] {
|
||||
@@ -32691,6 +32822,7 @@ export class OrcaRuntimeService {
|
||||
return {
|
||||
livePtyIds: targetedLiveness,
|
||||
allLivePtyIds: targetedLiveness,
|
||||
incarnationIdByPtyId: new Map(),
|
||||
terminalIdentityByPtyId: new Map(),
|
||||
queriedHostIds: new Set([LOCAL_EXECUTION_HOST_ID])
|
||||
}
|
||||
@@ -32762,11 +32894,21 @@ export class OrcaRuntimeService {
|
||||
const sessions = sessionsResult.value.processes
|
||||
const queriedHostIds = new Set(sessionsResult.value.hostIds)
|
||||
const controllerIdentityByPtyId = new Map<string, PtyControllerTerminalIdentity>()
|
||||
const incarnationIdByPtyId = new Map<string, string>()
|
||||
const ambiguousIncarnationPtyIds = new Set<string>()
|
||||
const ptyIdByControllerHandle = new Map<string, string>()
|
||||
const ambiguousControllerPtyIds = new Set<string>()
|
||||
for (const session of sessions) {
|
||||
const handle = session.terminalHandle?.trim()
|
||||
const incarnationId = session.incarnationId?.trim()
|
||||
if (incarnationId && !ambiguousIncarnationPtyIds.has(session.id)) {
|
||||
if (incarnationIdByPtyId.has(session.id)) {
|
||||
incarnationIdByPtyId.delete(session.id)
|
||||
ambiguousIncarnationPtyIds.add(session.id)
|
||||
} else {
|
||||
incarnationIdByPtyId.set(session.id, incarnationId)
|
||||
}
|
||||
}
|
||||
if (!handle?.startsWith('term_') || !incarnationId) {
|
||||
continue
|
||||
}
|
||||
@@ -32964,6 +33106,7 @@ export class OrcaRuntimeService {
|
||||
return {
|
||||
livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds,
|
||||
allLivePtyIds,
|
||||
incarnationIdByPtyId,
|
||||
terminalIdentityByPtyId: controllerIdentityByPtyId,
|
||||
queriedHostIds
|
||||
}
|
||||
|
||||
@@ -131,4 +131,94 @@ describe('PtyProcessLivenessBroker', () => {
|
||||
).resolves.toMatchObject({ status: 'live', foregroundProcess: 'zsh' })
|
||||
expect(inspectProcess).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('does not replay cached absence after owning inventory re-observes the PTY', async () => {
|
||||
let now = 0
|
||||
const inspectProcess = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('terminal_gone'))
|
||||
.mockRejectedValueOnce(new Error('socket_closed'))
|
||||
const source = { getForegroundProcess: vi.fn(async () => null), inspectProcess }
|
||||
const broker = new PtyProcessLivenessBroker({
|
||||
timeoutMs: 100,
|
||||
unavailableBackoffBaseMs: 100,
|
||||
now: () => now
|
||||
})
|
||||
|
||||
await expect(
|
||||
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
).resolves.toEqual({ status: 'exited' })
|
||||
await expect(
|
||||
broker.inspect({
|
||||
source,
|
||||
ptyId: 'pty-1',
|
||||
identity: 'incarnation-a',
|
||||
owningInventoryObservedPty: true
|
||||
})
|
||||
).resolves.toMatchObject({ status: 'unverifiable' })
|
||||
now = 201
|
||||
await expect(
|
||||
broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
).resolves.toEqual({ status: 'unverifiable', reason: 'socket_closed' })
|
||||
})
|
||||
|
||||
it('lets completion-sensitive readers await a shared probe past the catalog timeout', async () => {
|
||||
vi.useFakeTimers()
|
||||
let resolveInspection!: (value: {
|
||||
foregroundProcess: string
|
||||
hasChildProcesses: boolean
|
||||
}) => void
|
||||
const inspection = new Promise<{ foregroundProcess: string; hasChildProcesses: boolean }>(
|
||||
(resolve) => {
|
||||
resolveInspection = resolve
|
||||
}
|
||||
)
|
||||
const source = {
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess: vi.fn(() => inspection)
|
||||
}
|
||||
const broker = new PtyProcessLivenessBroker({ timeoutMs: 100 })
|
||||
|
||||
const catalog = broker.inspect({ source, ptyId: 'pty-1', identity: 'incarnation-a' })
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
await expect(catalog).resolves.toMatchObject({ status: 'unverifiable' })
|
||||
const completion = broker.inspect({
|
||||
source,
|
||||
ptyId: 'pty-1',
|
||||
identity: 'incarnation-a',
|
||||
waitForSettlement: true,
|
||||
reuseSettled: false
|
||||
})
|
||||
let settled = false
|
||||
void completion.then(() => {
|
||||
settled = true
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(settled).toBe(false)
|
||||
|
||||
resolveInspection({ foregroundProcess: 'codex', hasChildProcesses: true })
|
||||
await expect(completion).resolves.toMatchObject({ status: 'live', foregroundProcess: 'codex' })
|
||||
})
|
||||
|
||||
it('contains failures from diagnostic observers', async () => {
|
||||
const broker = new PtyProcessLivenessBroker({
|
||||
timeoutMs: 100,
|
||||
onInspectionError: () => {
|
||||
throw new Error('logger failed')
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
broker.inspect({
|
||||
source: {
|
||||
getForegroundProcess: vi.fn(async () => null),
|
||||
inspectProcess: vi.fn(async () => {
|
||||
throw new Error('socket_closed')
|
||||
})
|
||||
},
|
||||
ptyId: 'pty-1',
|
||||
identity: 'incarnation-a'
|
||||
})
|
||||
).resolves.toEqual({ status: 'unverifiable', reason: 'socket_closed' })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -21,6 +21,7 @@ type PtyProcessEvidenceEntry = {
|
||||
source: PtyProcessInspectionSource
|
||||
identity: string
|
||||
freshness: number
|
||||
owningInventoryObservedPty: boolean
|
||||
failureCount: number
|
||||
evidence: PtyProcessLivenessEvidence | null
|
||||
expiresAt: number
|
||||
@@ -56,16 +57,30 @@ export class PtyProcessLivenessBroker {
|
||||
freshness?: number
|
||||
reuseSettled?: boolean
|
||||
deadline?: number
|
||||
waitForSettlement?: boolean
|
||||
owningInventoryObservedPty?: boolean
|
||||
}): Promise<PtyProcessLivenessEvidence> {
|
||||
const freshness = args.freshness ?? 0
|
||||
const waitMs = Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
this.options.timeoutMs,
|
||||
args.deadline === undefined ? this.options.timeoutMs : args.deadline - this.now()
|
||||
)
|
||||
)
|
||||
const waitMs = args.waitForSettlement
|
||||
? null
|
||||
: Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
this.options.timeoutMs,
|
||||
args.deadline === undefined ? this.options.timeoutMs : args.deadline - this.now()
|
||||
)
|
||||
)
|
||||
const existing = this.entries.get(args.ptyId)
|
||||
if (
|
||||
existing?.source === args.source &&
|
||||
existing.identity === args.identity &&
|
||||
args.owningInventoryObservedPty === true
|
||||
) {
|
||||
existing.owningInventoryObservedPty = true
|
||||
if (existing.evidence?.status === 'exited') {
|
||||
this.storeUnverifiable(existing, 'owning inventory re-observed PTY')
|
||||
}
|
||||
}
|
||||
if (
|
||||
existing?.source === args.source &&
|
||||
existing.identity === args.identity &&
|
||||
@@ -90,7 +105,7 @@ export class PtyProcessLivenessBroker {
|
||||
existing.freshness >= freshness
|
||||
) {
|
||||
if (existing.probe) {
|
||||
return existing.timedOut
|
||||
return !args.waitForSettlement && existing.timedOut
|
||||
? Promise.resolve({ status: 'unverifiable', reason: 'process inspection timed out' })
|
||||
: waitMs === 0
|
||||
? Promise.resolve({
|
||||
@@ -115,6 +130,7 @@ export class PtyProcessLivenessBroker {
|
||||
source: args.source,
|
||||
identity: args.identity,
|
||||
freshness,
|
||||
owningInventoryObservedPty: args.owningInventoryObservedPty === true,
|
||||
failureCount,
|
||||
evidence: null,
|
||||
expiresAt: 0,
|
||||
@@ -123,23 +139,30 @@ export class PtyProcessLivenessBroker {
|
||||
}
|
||||
const probe = this.runProbe(args.source, args.ptyId)
|
||||
.then((evidence) => {
|
||||
const reconciled =
|
||||
evidence.status === 'exited' && entry.owningInventoryObservedPty
|
||||
? {
|
||||
status: 'unverifiable' as const,
|
||||
reason: 'owning inventory re-observed PTY'
|
||||
}
|
||||
: evidence
|
||||
if (this.entries.get(args.ptyId) !== entry) {
|
||||
return evidence
|
||||
return reconciled
|
||||
}
|
||||
entry.probe = null
|
||||
entry.timedOut = false
|
||||
entry.evidence = evidence
|
||||
if (evidence.status === 'unverifiable') {
|
||||
entry.evidence = reconciled
|
||||
if (reconciled.status === 'unverifiable') {
|
||||
entry.failureCount += 1
|
||||
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
|
||||
} else if (evidence.status === 'live') {
|
||||
} else if (reconciled.status === 'live') {
|
||||
entry.failureCount = 0
|
||||
entry.expiresAt = this.now() + (this.options.liveTtlMs ?? DEFAULT_LIVE_TTL_MS)
|
||||
} else {
|
||||
entry.failureCount = 0
|
||||
entry.expiresAt = Number.POSITIVE_INFINITY
|
||||
}
|
||||
return evidence
|
||||
return reconciled
|
||||
})
|
||||
.catch((error): PtyProcessLivenessEvidence => {
|
||||
if (this.entries.get(args.ptyId) === entry) {
|
||||
@@ -149,7 +172,11 @@ export class PtyProcessLivenessBroker {
|
||||
entry.evidence = { status: 'unverifiable', reason: describeError(error) }
|
||||
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
|
||||
}
|
||||
this.options.onInspectionError?.(args.ptyId, error)
|
||||
try {
|
||||
this.options.onInspectionError?.(args.ptyId, error)
|
||||
} catch {
|
||||
// Diagnostic observers cannot change the authority verdict.
|
||||
}
|
||||
return { status: 'unverifiable', reason: describeError(error) }
|
||||
})
|
||||
entry.probe = probe
|
||||
@@ -207,7 +234,7 @@ export class PtyProcessLivenessBroker {
|
||||
private waitForProbe(
|
||||
ptyId: string,
|
||||
entry: PtyProcessEvidenceEntry,
|
||||
timeoutMs: number
|
||||
timeoutMs: number | null
|
||||
): Promise<PtyProcessLivenessEvidence> {
|
||||
const probe = entry.probe
|
||||
if (!probe) {
|
||||
@@ -215,6 +242,9 @@ export class PtyProcessLivenessBroker {
|
||||
entry.evidence ?? { status: 'unverifiable', reason: 'process inspection unavailable' }
|
||||
)
|
||||
}
|
||||
if (timeoutMs === null) {
|
||||
return probe
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
if (this.entries.get(ptyId) === entry && entry.probe === probe) {
|
||||
@@ -234,6 +264,12 @@ export class PtyProcessLivenessBroker {
|
||||
const max = this.options.unavailableBackoffMaxMs ?? DEFAULT_UNAVAILABLE_BACKOFF_MAX_MS
|
||||
return Math.min(max, base * 2 ** Math.max(0, failureCount - 1))
|
||||
}
|
||||
|
||||
private storeUnverifiable(entry: PtyProcessEvidenceEntry, reason: string): void {
|
||||
entry.failureCount += 1
|
||||
entry.evidence = { status: 'unverifiable', reason }
|
||||
entry.expiresAt = this.now() + this.unavailableBackoffMs(entry.failureCount)
|
||||
}
|
||||
}
|
||||
|
||||
function isTerminalGoneError(error: unknown): boolean {
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
RestoredAgentAuthorityResolver,
|
||||
type RestoredAgentAuthorityBinding,
|
||||
type RestoredAgentAuthorityHook
|
||||
} from './restored-agent-authority-resolver'
|
||||
|
||||
const hook: RestoredAgentAuthorityHook = {
|
||||
identity: 'hook-a',
|
||||
paneKey: 'tab-a:leaf-a',
|
||||
worktreeKey: 'repo-a\0/worktree-a',
|
||||
hostKey: 'ssh:host-a'
|
||||
}
|
||||
|
||||
function binding(
|
||||
overrides: Partial<RestoredAgentAuthorityBinding> = {}
|
||||
): RestoredAgentAuthorityBinding {
|
||||
return {
|
||||
ptyId: 'pty-a',
|
||||
incarnationId: 'incarnation-a',
|
||||
lifecycleGeneration: 1,
|
||||
source: 'current',
|
||||
paneKey: hook.paneKey,
|
||||
worktreeKey: hook.worktreeKey,
|
||||
hostKey: hook.hostKey,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
describe('RestoredAgentAuthorityResolver', () => {
|
||||
it('rejects current process evidence from another execution host', () => {
|
||||
const resolver = new RestoredAgentAuthorityResolver()
|
||||
|
||||
expect(
|
||||
resolver.resolve({
|
||||
hook,
|
||||
current: binding({ hostKey: 'local' }),
|
||||
persisted: null
|
||||
})
|
||||
).toEqual({ binding: null, hasExactBinding: false })
|
||||
})
|
||||
|
||||
it('keeps one hook fenced to its first process incarnation', () => {
|
||||
const resolver = new RestoredAgentAuthorityResolver()
|
||||
expect(resolver.resolve({ hook, current: binding(), persisted: null }).binding).toEqual(
|
||||
binding()
|
||||
)
|
||||
|
||||
expect(
|
||||
resolver.resolve({
|
||||
hook,
|
||||
current: binding({ incarnationId: 'incarnation-b' }),
|
||||
persisted: null
|
||||
})
|
||||
).toEqual({ binding: null, hasExactBinding: false })
|
||||
})
|
||||
|
||||
it('treats lifecycle generations as fallback identity when incarnation exists', () => {
|
||||
const resolver = new RestoredAgentAuthorityResolver()
|
||||
resolver.resolve({ hook, current: binding(), persisted: null })
|
||||
|
||||
expect(
|
||||
resolver.resolve({
|
||||
hook,
|
||||
current: binding({ lifecycleGeneration: 2 }),
|
||||
persisted: null
|
||||
}).binding
|
||||
).toEqual(binding({ lifecycleGeneration: 2 }))
|
||||
})
|
||||
|
||||
it('rejects disagreement between current and persisted bindings', () => {
|
||||
const resolver = new RestoredAgentAuthorityResolver()
|
||||
|
||||
expect(
|
||||
resolver.resolve({
|
||||
hook,
|
||||
current: binding({ incarnationId: 'incarnation-b' }),
|
||||
persisted: binding({ source: 'persisted' })
|
||||
})
|
||||
).toEqual({ binding: null, hasExactBinding: false })
|
||||
})
|
||||
|
||||
it('forgets commitments after their restored hook disappears', () => {
|
||||
const resolver = new RestoredAgentAuthorityResolver()
|
||||
resolver.resolve({ hook, current: binding(), persisted: null })
|
||||
resolver.retain(new Set())
|
||||
|
||||
expect(
|
||||
resolver.resolve({
|
||||
hook,
|
||||
current: binding({ incarnationId: 'incarnation-b' }),
|
||||
persisted: null
|
||||
}).binding
|
||||
).toEqual(binding({ incarnationId: 'incarnation-b' }))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,89 @@
|
||||
export type RestoredAgentAuthorityHook = Readonly<{
|
||||
identity: string
|
||||
paneKey: string
|
||||
worktreeKey: string
|
||||
hostKey: string
|
||||
terminalHandle?: string
|
||||
}>
|
||||
|
||||
export type RestoredAgentAuthorityBinding = Readonly<{
|
||||
ptyId: string
|
||||
incarnationId: string | null
|
||||
lifecycleGeneration: number
|
||||
source: 'current' | 'persisted'
|
||||
paneKey: string
|
||||
worktreeKey: string
|
||||
hostKey: string
|
||||
terminalHandle?: string
|
||||
}>
|
||||
|
||||
export type RestoredAgentAuthorityResolution = Readonly<{
|
||||
binding: RestoredAgentAuthorityBinding | null
|
||||
hasExactBinding: boolean
|
||||
}>
|
||||
|
||||
export class RestoredAgentAuthorityResolver {
|
||||
private readonly commitmentByHookIdentity = new Map<string, string>()
|
||||
|
||||
resolve(args: {
|
||||
hook: RestoredAgentAuthorityHook
|
||||
current: RestoredAgentAuthorityBinding | null
|
||||
persisted: RestoredAgentAuthorityBinding | null
|
||||
}): RestoredAgentAuthorityResolution {
|
||||
const current = this.matchesHook(args.hook, args.current) ? args.current : null
|
||||
const persisted = this.matchesHook(args.hook, args.persisted) ? args.persisted : null
|
||||
if (current && persisted && !sameRestoredAgentProcessAuthority(current, persisted)) {
|
||||
return { binding: null, hasExactBinding: false }
|
||||
}
|
||||
const binding = current ?? persisted
|
||||
if (!binding) {
|
||||
return { binding: null, hasExactBinding: false }
|
||||
}
|
||||
const commitment = processAuthorityKey(binding)
|
||||
const existing = this.commitmentByHookIdentity.get(args.hook.identity)
|
||||
if (existing && existing !== commitment) {
|
||||
return { binding: null, hasExactBinding: false }
|
||||
}
|
||||
this.commitmentByHookIdentity.set(args.hook.identity, commitment)
|
||||
return { binding, hasExactBinding: true }
|
||||
}
|
||||
|
||||
retain(hookIdentities: ReadonlySet<string>): void {
|
||||
for (const identity of this.commitmentByHookIdentity.keys()) {
|
||||
if (!hookIdentities.has(identity)) {
|
||||
this.commitmentByHookIdentity.delete(identity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private matchesHook(
|
||||
hook: RestoredAgentAuthorityHook,
|
||||
binding: RestoredAgentAuthorityBinding | null
|
||||
): binding is RestoredAgentAuthorityBinding {
|
||||
return Boolean(
|
||||
binding &&
|
||||
binding.paneKey === hook.paneKey &&
|
||||
binding.worktreeKey === hook.worktreeKey &&
|
||||
binding.hostKey === hook.hostKey &&
|
||||
(!hook.terminalHandle || binding.terminalHandle === hook.terminalHandle)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function sameRestoredAgentProcessAuthority(
|
||||
left: RestoredAgentAuthorityBinding,
|
||||
right: RestoredAgentAuthorityBinding
|
||||
): boolean {
|
||||
return processAuthorityKey(left) === processAuthorityKey(right)
|
||||
}
|
||||
|
||||
function processAuthorityKey(binding: RestoredAgentAuthorityBinding): string {
|
||||
return JSON.stringify([
|
||||
binding.hostKey,
|
||||
binding.worktreeKey,
|
||||
binding.paneKey,
|
||||
binding.ptyId,
|
||||
binding.incarnationId ?? `generation:${binding.lifecycleGeneration}`,
|
||||
binding.terminalHandle ?? null
|
||||
])
|
||||
}
|
||||
Reference in New Issue
Block a user