From 09fe2aae49dd0df30a78efe98b5dfea1a4627006 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:37:27 -0700 Subject: [PATCH] fix(ssh): preserve unconfirmed stop verdicts across consumers --- src/main/ipc/pty-ssh-stop-verdict.test.ts | 54 +++++++++++- src/main/ipc/pty.ts | 13 +++ .../claude-agent-teams-service.test.ts | 43 ++++++++++ .../claude-agent-teams-tmux-dispatcher.ts | 16 +++- ...-runtime-terminal-close-continuity.test.ts | 36 ++++++-- src/main/runtime/orca-runtime.ts | 80 ++++++++++++++---- .../pty-inventory-liveness-verdict.test.ts | 78 ++++++++++++++++- .../orchestration-federation-control.ts | 8 +- ...ration-federation-liveness-verdict.test.ts | 35 +++++++- .../orchestration-federation-output.test.ts | 2 +- .../orchestration-federation-setup.test.ts | 1 + .../methods/orchestration-federation.test.ts | 5 +- .../methods/orchestration-worker-control.ts | 6 +- .../orchestration-worker-observation.ts | 6 +- ...orchestration-worker-release-completion.ts | 15 +++- ...on-worker-release-liveness-verdict.test.ts | 84 +++++++++++++++++++ .../orchestration-worker-release.test.ts | 3 +- ...ation-worker-stop-liveness-verdict.test.ts | 43 ++++++++++ .../rpc/methods/orchestration-worker-stop.ts | 4 +- .../orchestration-workers-recovery.test.ts | 5 +- .../runtime/unstopped-pty-verification.ts | 14 ++++ src/main/runtime/worktree-teardown.test.ts | 47 ++++++++++- src/main/runtime/worktree-teardown.ts | 23 +++-- src/shared/pty-liveness-verdict.ts | 2 +- 24 files changed, 562 insertions(+), 61 deletions(-) create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts diff --git a/src/main/ipc/pty-ssh-stop-verdict.test.ts b/src/main/ipc/pty-ssh-stop-verdict.test.ts index f22dc825a52..aee780e8a37 100644 --- a/src/main/ipc/pty-ssh-stop-verdict.test.ts +++ b/src/main/ipc/pty-ssh-stop-verdict.test.ts @@ -4,7 +4,9 @@ import { registerPtyHandlers, deletePtyOwnership, setPtyOwnership, - getLocalPtyProvider + getLocalPtyProvider, + registerSshPtyProvider, + unregisterSshPtyProvider } from './pty' vi.mock('electron', () => import('./pty-ipc-mock-registry').then((m) => m.electronModuleMock())) @@ -65,12 +67,14 @@ describe('stopping a PTY whose SSH provider is unregistered', () => { setPtyController: ReturnType onPtyExit: ReturnType markPtyLivenessUnverifiable: ReturnType + markPtyLivenessLive: ReturnType } } { const runtime = { setPtyController: vi.fn(), onPtyExit: vi.fn(), - markPtyLivenessUnverifiable: vi.fn() + markPtyLivenessUnverifiable: vi.fn(), + markPtyLivenessLive: vi.fn() } handlers.clear() registerPtyHandlers(mainWindow as never, runtime as never) @@ -119,4 +123,50 @@ describe('stopping a PTY whose SSH provider is unregistered', () => { ) expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled() }) + + it('reports lost contact when a registered provider drops during the stop', async () => { + const connectionId = 'ssh-mid-stop-drop' + const ptyId = 'ssh-mid-stop-pty' + const provider = { + onExit: vi.fn(() => () => {}), + shutdown: vi.fn(async () => { + throw new Error('relay disconnected during stop') + }) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + + await expect(controller.stopAndWait(ptyId)).resolves.toBe(false) + expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith( + ptyId, + 'relay disconnected during stop' + ) + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) + + it('reports a provider-observed survivor as live', async () => { + const connectionId = 'ssh-still-live' + const ptyId = 'ssh-still-live-pty' + const provider = { + onExit: vi.fn(() => () => {}), + shutdown: vi.fn(async () => {}), + listProcesses: vi.fn(async () => [{ id: ptyId }]) + } + registerSshPtyProvider(connectionId, provider as never) + setPtyOwnership(ptyId, connectionId) + try { + const { controller, runtime } = installController() + + await expect(controller.stopAndWait(ptyId)).resolves.toBe(false) + expect(runtime.markPtyLivenessLive).toHaveBeenCalledWith(ptyId) + } finally { + deletePtyOwnership(ptyId) + unregisterSshPtyProvider(connectionId) + } + }) }) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 0ef5ceaff60..58f8559396b 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -5672,6 +5672,12 @@ export function registerPtyHandlers( }) } catch (err) { if (!isPtyAlreadyGoneError(err)) { + if (connectionId) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + err instanceof Error ? err.message : String(err) + ) + } console.warn( `[pty] Failed to stop PTY ${ptyId}: ${err instanceof Error ? err.message : String(err)}` ) @@ -5680,9 +5686,16 @@ export function registerPtyHandlers( } try { if (!(await verifyPtyStopped(provider, ptyId, opts))) { + runtime?.markPtyLivenessLive?.(ptyId) return false } } catch (err) { + if (connectionId) { + runtime?.markPtyLivenessUnverifiable?.( + ptyId, + err instanceof Error ? err.message : String(err) + ) + } console.warn( `[pty] Failed to verify PTY ${ptyId} stopped: ${ err instanceof Error ? err.message : String(err) diff --git a/src/main/runtime/claude-agent-teams-service.test.ts b/src/main/runtime/claude-agent-teams-service.test.ts index a41668932f0..8d65370b91e 100644 --- a/src/main/runtime/claude-agent-teams-service.test.ts +++ b/src/main/runtime/claude-agent-teams-service.test.ts @@ -218,6 +218,49 @@ describe('ClaudeAgentTeamsService', () => { expect(api.closeTerminal).toHaveBeenCalledWith('teammate-1') }) + it('keeps a pane registered when its process stop is unconfirmed', async () => { + const { service, teamId, token, leaderPane, api } = createServiceWithLeader() + const request = (argv: string[], envPane = leaderPane) => + service.handleTmuxCompat({ teamId, token, envPane, argv }, api) + + await request(['split-window', '-t', leaderPane, '-h', '-P', '-F', '#{pane_id}']) + vi.mocked(api.closeTerminal).mockResolvedValueOnce({ + handle: 'teammate-1', + tabId: 'tab-1', + ptyKilled: false + }) + + await expect(request(['kill-pane', '-t', '%2'])).resolves.toMatchObject({ + ok: false, + exitCode: 1 + }) + await expect( + request(['list-panes', '-t', 'orca:0', '-F', '#{pane_id}']) + ).resolves.toMatchObject({ stdout: '%1\n%2\n' }) + }) + + it('keeps the replacement addressable when the retired placeholder stop is unconfirmed', async () => { + const { service, teamId, token, leaderPane, api } = createServiceWithLeader() + const request = (argv: string[], envPane = leaderPane) => + service.handleTmuxCompat({ teamId, token, envPane, argv }, api) + + await request(['split-window', '-t', leaderPane, '-h', '-P', '-F', '#{pane_id}', 'cat']) + vi.mocked(api.closeTerminal).mockResolvedValueOnce({ + handle: 'teammate-1', + tabId: 'tab-1', + ptyKilled: false, + ptyStopVerdict: 'live' + }) + + await expect( + request(['respawn-pane', '-k', '-t', '%2', '--', 'claude --agent-id a']) + ).resolves.toMatchObject({ ok: false, exitCode: 1 }) + expect(api.closeTerminal).toHaveBeenNthCalledWith(1, 'teammate-1') + + await request(['kill-pane', '-t', '%2']) + expect(api.closeTerminal).toHaveBeenLastCalledWith('teammate-2') + }) + it('refuses to respawn the leader pane', async () => { const { service, teamId, token, leaderPane, api } = createServiceWithLeader() diff --git a/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts b/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts index 98159e8725e..d6ab2c65ea7 100644 --- a/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts +++ b/src/main/runtime/claude-agent-teams-tmux-dispatcher.ts @@ -4,6 +4,7 @@ import { tmuxSendKeysText, tmuxValue } from '../../shared/claude-agent-teams-tmux-compat' +import { describeUnconfirmedAgentStop } from '../../shared/pty-liveness-verdict' import { formatContext, paneEnv, @@ -172,9 +173,15 @@ export class ClaudeAgentTeamsTmuxDispatcher { activate: false }) try { - await api.closeTerminal(previousHandle) + const close = await api.closeTerminal(previousHandle) + if (!close.ptyKilled) { + pane.handle = split.handle + throw new Error(describeUnconfirmedAgentStop(close)) + } } catch (error) { - await api.closeTerminal(split.handle).catch(() => {}) + if (pane.handle === previousHandle) { + await api.closeTerminal(split.handle).catch(() => {}) + } throw error } pane.handle = split.handle @@ -266,7 +273,10 @@ export class ClaudeAgentTeamsTmuxDispatcher { if (pane.fakePaneId === team.leaderPane) { throw new Error('refusing to kill leader pane') } - await api.closeTerminal(pane.handle) + const close = await api.closeTerminal(pane.handle) + if (!close.ptyKilled) { + throw new Error(describeUnconfirmedAgentStop(close)) + } team.panes.delete(pane.fakePaneId) team.paneOrder = team.paneOrder.filter((id) => id !== pane.fakePaneId) if (team.mainVertical?.lastColumnPane === pane.fakePaneId) { diff --git a/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts b/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts index 0f36529663d..a4a11b11a31 100644 --- a/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts +++ b/src/main/runtime/orca-runtime-terminal-close-continuity.test.ts @@ -311,7 +311,7 @@ describe('terminal close and handle incarnation continuity', () => { harness.retirePersistedTab() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: false }) expect(harness.kill).toHaveBeenCalledWith(PTY_ID) expect(harness.closeTerminal).not.toHaveBeenCalled() expect(harness.getSession().tabsByWorktree[WORKTREE_ID]).toEqual([]) @@ -328,7 +328,7 @@ describe('terminal close and handle incarnation continuity', () => { expect(harness.getSession().tabsByWorktree[WORKTREE_ID]).toHaveLength(1) }) - it('kills every live tab PTY after retirement when the renderer graph is stale', async () => { + it('requests a stop for every live tab PTY after retirement when the renderer graph is stale', async () => { const harness = createHarness() harness.syncSplitFixtureGraph() const terminal = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals.find( @@ -342,7 +342,7 @@ describe('terminal close and handle incarnation continuity', () => { harness.retirePersistedTab() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ ptyKilled: false }) expect(harness.kill).toHaveBeenCalledWith(PTY_ID) expect(harness.kill).toHaveBeenCalledWith(SIBLING_PTY_ID) }) @@ -386,9 +386,29 @@ describe('terminal close and handle incarnation continuity', () => { }) }) + it('downgrades a live verdict after issuing an unverified follow-up stop', async () => { + const harness = createHarness() + const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals + harness.setVerifiedStopResult(false) + harness.runtime.markPtyLivenessLive(PTY_ID) + + const closing = harness.runtime.closeTerminal(handle) + await vi.waitFor(() => expect(harness.closeTerminalTab).toHaveBeenCalled()) + harness.retirePersistedTab() + harness.acknowledged.resolve() + + await expect(closing).resolves.toMatchObject({ + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'a follow-up stop was issued but its outcome could not be verified' + }) + expect(harness.kill).toHaveBeenCalledWith(PTY_ID) + }) + it('leaves a confirmed kill receipt free of any stop verdict', async () => { const harness = createHarness() const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals + harness.setVerifiedStopResult(true) const closing = harness.runtime.closeTerminal(handle) await vi.waitFor(() => expect(harness.closeTerminalTab).toHaveBeenCalled()) @@ -401,7 +421,7 @@ describe('terminal close and handle incarnation continuity', () => { expect(close.ptyStopReason).toBeUndefined() }) - it('falls back to kill when verified teardown rejects after retirement', async () => { + it('reports an unconfirmed stop when verified teardown rejects after retirement', async () => { const harness = createHarness() const [{ handle }] = (await harness.runtime.listTerminals(`id:${WORKTREE_ID}`)).terminals harness.setVerifiedStopResult(new Error('provider_unavailable')) @@ -411,7 +431,11 @@ describe('terminal close and handle incarnation continuity', () => { harness.retirePersistedTab() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ + ptyKilled: false, + ptyStopVerdict: 'unverifiable', + ptyStopReason: 'provider_unavailable' + }) expect(harness.stopAndWait).toHaveBeenCalledWith(PTY_ID, { deadlineMs: expect.any(Number) }) @@ -434,7 +458,7 @@ describe('terminal close and handle incarnation continuity', () => { harness.makeSessionUnavailable() harness.acknowledged.resolve() - await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: true }) + await expect(closing).resolves.toMatchObject({ handle, tabId: TAB_ID, ptyKilled: false }) expect(harness.closeTerminal).toHaveBeenCalledWith(TAB_ID) expect(harness.stopAndWait).toHaveBeenCalledWith(RUNTIME_OWNED_PTY_ID, { deadlineMs: expect.any(Number) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 2b399e71353..1256f128b9c 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -1557,9 +1557,14 @@ function isAgentSessionOperationOutcomeUnknown(error: unknown): boolean { ) } -// Bounded because verdicts outlive their pty records; oldest-first eviction. +// Orphaned verdicts are bounded; active PTYs retain theirs until new evidence resolves them. const MAX_TRACKED_PTY_LIVENESS_VERDICTS = 256 +type TrackedPtyLivenessVerdict = { + verdict: PtyLivenessVerdict + observedAt: number +} + const AGENT_SESSION_OPERATION_PER_CLIENT_LIMIT = 512 const AGENT_SESSION_OPERATION_GLOBAL_LIMIT = 4_096 @@ -3136,7 +3141,8 @@ export class OrcaRuntimeService { // clears, so it cannot distinguish an observed exit from lost contact. This // records the last liveness verdict we actually earned, and outlives the pty // record so a close/stop receipt can still say the stop was unconfirmed. - private ptyLivenessVerdictByPtyId = new Map() + private ptyLivenessVerdictByPtyId = new Map() + private ptyLivenessObservationSequence = 0 private readonly pairedRendererSessionOwnedPtyIds = new Set() private wslDistroByPtyId = new Map() private titleObservationSequence = 0 @@ -10458,6 +10464,7 @@ export class OrcaRuntimeService { incarnationId?: PtyIncarnationId, options: { awaitsRegistration?: boolean } = {} ): void { + this.forgetPtyLivenessVerdict(ptyId) if (options.awaitsRegistration !== false) { // Why: surface absence cannot distinguish an in-flight admission from a completed headless lifecycle. this.pendingPtyRegistrationIncarnations.set(ptyId, incarnationId ?? null) @@ -10491,6 +10498,7 @@ export class OrcaRuntimeService { isWsl?: boolean ): void { this.assertPtyDidNotExitBeforeRegistration(ptyId, binding?.incarnationId) + this.forgetPtyLivenessVerdict(ptyId) this.spawnPublishedPtys.add(ptyId) // Why: record the renderer pane identity at spawn time so a stalled graph // sync can't hide that a live PTY already backs a pending mobile create. @@ -17544,9 +17552,13 @@ export class OrcaRuntimeService { this.rememberPtyLivenessVerdict(ptyId, { status: 'unverifiable', reason }) } + markPtyLivenessLive(ptyId: string): void { + this.rememberPtyLivenessVerdict(ptyId, { status: 'live', ptyIds: [ptyId] }) + } + /** Null when nothing has been observed either way, so callers keep their own default. */ getPtyLivenessVerdict(ptyId: string): PtyLivenessVerdict | null { - return this.ptyLivenessVerdictByPtyId.get(ptyId) ?? null + return this.ptyLivenessVerdictByPtyId.get(ptyId)?.verdict ?? null } getTerminalLivenessVerdict(handle: string): PtyLivenessVerdict | null { @@ -17561,17 +17573,35 @@ export class OrcaRuntimeService { return } this.ptyLivenessVerdictByPtyId.delete(ptyId) - this.ptyLivenessVerdictByPtyId.set(ptyId, verdict) + this.ptyLivenessObservationSequence += 1 + this.ptyLivenessVerdictByPtyId.set(ptyId, { + verdict, + observedAt: this.ptyLivenessObservationSequence + }) while (this.ptyLivenessVerdictByPtyId.size > MAX_TRACKED_PTY_LIVENESS_VERDICTS) { - const oldest = this.ptyLivenessVerdictByPtyId.keys().next() - if (oldest.done) { + let oldestOrphaned: string | null = null + for (const candidate of this.ptyLivenessVerdictByPtyId.keys()) { + if ( + !this.ptysById.has(candidate) && + !this.handleByPtyId.has(candidate) && + !this.leafExistsForPty(candidate) + ) { + oldestOrphaned = candidate + break + } + } + if (!oldestOrphaned) { return } - this.ptyLivenessVerdictByPtyId.delete(oldest.value) + this.ptyLivenessVerdictByPtyId.delete(oldestOrphaned) } } - private forgetPtyLivenessVerdict(ptyId: string): void { + private forgetPtyLivenessVerdict(ptyId: string, observedNoLaterThan?: number): void { + const tracked = this.ptyLivenessVerdictByPtyId.get(ptyId) + if (observedNoLaterThan !== undefined && tracked && tracked.observedAt > observedNoLaterThan) { + return + } this.ptyLivenessVerdictByPtyId.delete(ptyId) } @@ -28491,13 +28521,29 @@ export class OrcaRuntimeService { let addressedPtyStopped = false const deadlineMs = Date.now() + EXPLICIT_TERMINAL_CLOSE_STOP_TIMEOUT_MS for (const ptyId of ptyIds) { - let verifiedStopped = false - try { - verifiedStopped = (await this.ptyController?.stopAndWait?.(ptyId, { deadlineMs })) ?? false - } catch { - // Why: verified teardown is preferred, but its transport failure must not suppress the legacy provider kill. + let stopped = false + if (this.ptyController?.stopAndWait) { + try { + stopped = await this.ptyController.stopAndWait(ptyId, { deadlineMs }) + } catch (error) { + this.markPtyLivenessUnverifiable( + ptyId, + error instanceof Error ? error.message : String(error) + ) + } + if (!stopped) { + this.ptyController.kill(ptyId) + const verdict = this.getPtyLivenessVerdict(ptyId) + if (!verdict || verdict.status === 'live') { + this.markPtyLivenessUnverifiable( + ptyId, + 'a follow-up stop was issued but its outcome could not be verified' + ) + } + } + } else { + stopped = this.ptyController?.kill(ptyId) ?? false } - const stopped = verifiedStopped || (this.ptyController?.kill(ptyId) ?? false) if (ptyId === addressedPtyId) { addressedPtyStopped = stopped } @@ -28707,7 +28753,7 @@ export class OrcaRuntimeService { const ptyKilled = await this.stopExplicitlyClosedTabPtys(ptyIdsToKill, pty.pty.ptyId) return this.describeTerminalClose(handle, tabId, pty.pty.ptyId, ptyKilled) } - const ptyKilled = this.ptyController?.kill(pty.pty.ptyId) ?? false + const ptyKilled = await this.stopExplicitlyClosedTabPtys([pty.pty.ptyId], pty.pty.ptyId) if (!ptyKilled || siblingCount <= 1) { if (surface) { // Why: paired viewers keep ended streams mounted until the HUB publishes removal, so explicit close uses the durable host-tab transaction instead of viewer-local exit handling. @@ -31325,6 +31371,7 @@ export class OrcaRuntimeService { } const inventoryGeneration = this.ptyControllerInventorySequence + 1 this.ptyControllerInventorySequence = inventoryGeneration + const livenessObservationAtStart = this.ptyLivenessObservationSequence const providerKey = typeof connectionId === 'string' ? `ssh:${connectionId}` : 'local' if (connectionId === undefined) { this.ptyControllerAggregateInventoryGeneration = inventoryGeneration @@ -31409,6 +31456,8 @@ export class OrcaRuntimeService { const allLivePtyIds = new Set(sessions.map((session) => session.id)) const selectedLivePtyIds = new Set() for (const session of sessions) { + // The owning inventory positively observed this PTY again; prior lost-contact doubt is stale. + this.forgetPtyLivenessVerdict(session.id, livenessObservationAtStart) const sessionConnectionId = parseAppSshPtyId(session.id)?.connectionId ?? (typeof connectionId === 'string' ? connectionId : null) @@ -31616,6 +31665,7 @@ export class OrcaRuntimeService { if (pty) { pty.connected = true pty.disconnectedAt = null + this.forgetPtyLivenessVerdict(ptyId) this.refreshPtyForegroundAgent(ptyId) } } else if (pty && !this.leafExistsForPty(ptyId)) { diff --git a/src/main/runtime/pty-inventory-liveness-verdict.test.ts b/src/main/runtime/pty-inventory-liveness-verdict.test.ts index 61b73c884e3..4034933b1d6 100644 --- a/src/main/runtime/pty-inventory-liveness-verdict.test.ts +++ b/src/main/runtime/pty-inventory-liveness-verdict.test.ts @@ -9,6 +9,14 @@ import { getDefaultWorkspaceSession } from '../../shared/constants' const WORKTREE_ID = 'repo-1::/tmp/inventory-verdict' const REMOTE_PTY_ID = 'ssh:conn-1@@relay-9' +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + function makeStore() { const session = getDefaultWorkspaceSession() return { @@ -32,13 +40,16 @@ function makeStore() { } } -function makeRuntimeMissingFromInventory(hasPty: () => boolean | null): OrcaRuntimeService { +function makeRuntimeMissingFromInventory( + hasPty: () => boolean | null, + listProcesses: () => Promise<{ id: string; worktreeId: string }[]> = vi.fn(async () => []) +): OrcaRuntimeService { const runtime = new OrcaRuntimeService(makeStore() as never) runtime.setPtyController({ write: () => true, kill: () => true, hasPty, - listProcesses: vi.fn(async () => []), + listProcesses, getForegroundProcess: async () => null } as never) runtime.attachWindow(1) @@ -67,4 +78,67 @@ describe('inventory sweep liveness verdicts', () => { // An observed absence is the death certificate callers already act on. expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() }) + + it('clears lost-contact doubt when reconnect inventory observes the PTY live', async () => { + let reconnected = false + const runtime = makeRuntimeMissingFromInventory( + () => null, + vi.fn(async () => (reconnected ? [{ id: REMOTE_PTY_ID, worktreeId: WORKTREE_ID }] : [])) + ) + + await runtime.listTerminals(`id:${WORKTREE_ID}`) + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)?.status).toBe('unverifiable') + + reconnected = true + await runtime.listTerminals(`id:${WORKTREE_ID}`) + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + }) + + it('does not let a pre-drop inventory clear a newer lost-contact verdict', async () => { + const inventory = deferred<{ id: string; worktreeId: string }[]>() + const listProcesses = vi.fn(() => inventory.promise) + const runtime = makeRuntimeMissingFromInventory(() => null, listProcesses) + + const listing = runtime.listTerminals(`id:${WORKTREE_ID}`) + await vi.waitFor(() => expect(listProcesses).toHaveBeenCalled()) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'relay disconnected during stop') + inventory.resolve([{ id: REMOTE_PTY_ID, worktreeId: WORKTREE_ID }]) + await listing + + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toEqual({ + status: 'unverifiable', + reason: 'relay disconnected during stop' + }) + }) + + it('clears stale doubt when a new PTY lifecycle is positively registered', () => { + const runtime = makeRuntimeMissingFromInventory(() => null) + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'old incarnation lost contact') + + runtime.onPtySpawned(REMOTE_PTY_ID, 'incarnation-2') + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + + runtime.markPtyLivenessUnverifiable(REMOTE_PTY_ID, 'registration raced reconnect') + runtime.registerPty(REMOTE_PTY_ID, WORKTREE_ID, 'conn-1', { + tabId: 'tab-new', + leafId: '00000000-0000-4000-8000-000000000001', + incarnationId: 'incarnation-2' + }) + expect(runtime.getPtyLivenessVerdict(REMOTE_PTY_ID)).toBeNull() + }) + + it('retains unresolved verdicts for every still-addressable PTY', () => { + const runtime = new OrcaRuntimeService(makeStore() as never) + for (let index = 0; index < 257; index += 1) { + const ptyId = `ssh:conn-1@@relay-${index}` + runtime.registerPty(ptyId, WORKTREE_ID, 'conn-1') + runtime.markPtyLivenessUnverifiable(ptyId, 'provider disconnected') + } + + expect(runtime.getPtyLivenessVerdict('ssh:conn-1@@relay-0')).toEqual({ + status: 'unverifiable', + reason: 'provider disconnected' + }) + }) }) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-control.ts b/src/main/runtime/rpc/methods/orchestration-federation-control.ts index 4ae00185c4a..4f186fb7cc2 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-control.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-control.ts @@ -51,7 +51,7 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ handler: async (params, { runtime, authenticatedCallerFingerprint }) => { requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) const observation = await inspectRemoteAttachment(runtime, params.dispatchId) - // Why `=== 'exited'` rather than `!== 'running'`: the other non-running + // Why `=== 'exited'` rather than `!== 'live'`: the other non-live // statuses are already covered by the two guards, and an unverifiable // terminal is still readable — losing stop-contact is not an exit. if (!observation.exact || !observation.terminal || observation.status === 'exited') { @@ -142,7 +142,7 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ } try { const close = await runtime.closeTerminal(observation.terminal.handle) - if (close.ptyStopVerdict) { + if (!close.ptyKilled) { // The tab is retired but the process was never confirmed stopped, so // the coordinator must not be told this dispatch reached 'stopped'. const attachment = db.markRemoteAttachmentStopUnknown( @@ -202,7 +202,7 @@ async function inspectRemoteAttachment( ): Promise<{ terminal: Awaited> | null exact: boolean - status: 'unattached' | 'missing' | 'identity_changed' | 'running' | 'exited' | 'unverifiable' + status: 'unattached' | 'missing' | 'identity_changed' | 'live' | 'exited' | 'unverifiable' /** Set with `unverifiable`; names what we lost contact with. */ reason?: string }> { @@ -233,7 +233,7 @@ async function inspectRemoteAttachment( return { terminal, exact, - status: verdict?.status !== 'live' && terminal.connected === false ? 'exited' : 'running' + status: verdict?.status !== 'live' && terminal.connected === false ? 'exited' : 'live' } } diff --git a/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts index ffa2ba99864..7141acc5d2c 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-liveness-verdict.test.ts @@ -82,6 +82,23 @@ describe('federation host liveness verdicts', () => { }) }) + it('uses the canonical live verdict for an observed running process', async () => { + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: HANDLE, + worktreeId: 'repo::remote-worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'live', + ptyIds: [HANDLE] + }) + + await expect( + call('orchestration.federationShow', { dispatchId: DISPATCH_ID }) + ).resolves.toMatchObject({ observation: { status: 'live', exactWorker: true } }) + }) + it('still reports a locally observed exit as exited', async () => { await expect( call('orchestration.federationShow', { dispatchId: DISPATCH_ID }) @@ -89,7 +106,7 @@ describe('federation host liveness verdicts', () => { }) it('still serves output for a terminal we merely lost stop-contact with', async () => { - // Why this matters: the read gate used to be `status !== 'running'`, which + // Why this matters: the read gate used to reject every status except live, which // would refuse a connected terminal the moment a stop lost contact with it. vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ handle: HANDLE, @@ -132,6 +149,22 @@ describe('federation host liveness verdicts', () => { expect(stopped.lastError).toContain('could not be confirmed stopped') }) + it('does not settle a bare false close as a stop', async () => { + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: HANDLE, + tabId: 'tab_remote', + ptyKilled: false + }) + + const stopped = (await call('orchestration.federationStop', { dispatchId: DISPATCH_ID })) as { + state: string + lastError?: string + } + + expect(stopped.state).not.toBe('stopped') + expect(stopped.lastError).toContain('could not be confirmed stopped') + }) + it('still settles a confirmed close as a stop', async () => { vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: HANDLE, diff --git a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts index 9617c05c125..adb2cc5459a 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts @@ -192,7 +192,7 @@ describe('orchestration federated worker output', () => { ok: true, result: { server: { environmentId: 'environment_windows', name: 'windows' }, - observation: { status: 'running', exactWorker: true }, + observation: { status: 'live', exactWorker: true }, terminal: { handle: 'term_windows_worker' } } }) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts index 70c08e2ce95..6eec52cd93f 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts @@ -181,6 +181,7 @@ describe('orchestration federated setup evidence', () => { await expect( workerShow.handler(workerShow.params!.parse({ dispatch: started.dispatch.id }), { runtime }) ).resolves.toMatchObject({ + observation: { status: 'live', exactWorker: true }, worker: { state: 'ready', stage: 'input_accepted', diff --git a/src/main/runtime/rpc/methods/orchestration-federation.test.ts b/src/main/runtime/rpc/methods/orchestration-federation.test.ts index 18db8750ff0..59dd8a7a5f8 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation.test.ts @@ -152,7 +152,8 @@ describe('orchestration federation', () => { } as never) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_windows_worker', - closed: true + tabId: 'tab-windows-worker', + ptyKilled: true } as never) } @@ -664,7 +665,7 @@ describe('orchestration federation', () => { expect(shown).toMatchObject({ ok: true, - result: { observation: { status: 'running', exactWorker: true } } + result: { observation: { status: 'live', exactWorker: true } } }) expect(homeDb.getFederatedDispatch(dispatch.id)?.remote_runtime_epoch).not.toBe(oldEpoch) expect(homeDb.getFederatedDispatch(dispatch.id)?.peer_fingerprint).toBe( diff --git a/src/main/runtime/rpc/methods/orchestration-worker-control.ts b/src/main/runtime/rpc/methods/orchestration-worker-control.ts index 5ccb7faff59..cc8edfaf8f8 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-control.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-control.ts @@ -99,7 +99,11 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [ server: { environmentId: server.environmentId, name: server.name }, remoteRuntimeEpoch: remote.runtimeEpoch, terminal: remote.terminal, - observation: remote.observation + observation: { + ...remote.observation, + // Legacy servers published `running`; normalize at the compatibility boundary. + status: remote.observation.status === 'running' ? 'live' : remote.observation.status + } } } if (worker.runtime_epoch && worker.runtime_epoch !== runtime.getRuntimeId()) { diff --git a/src/main/runtime/rpc/methods/orchestration-worker-observation.ts b/src/main/runtime/rpc/methods/orchestration-worker-observation.ts index 75c6eb77665..fe6ce087b7f 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-observation.ts @@ -10,7 +10,7 @@ export async function inspectWorkerTerminal( ): Promise<{ terminal: Awaited> | null exact: boolean - status: 'unattached' | 'missing' | 'identity_changed' | 'running' | 'exited' | 'unverifiable' + status: 'unattached' | 'missing' | 'identity_changed' | 'live' | 'exited' | 'unverifiable' /** Set with `unverifiable`; names what we lost contact with. */ reason?: string }> { @@ -38,12 +38,12 @@ export async function inspectWorkerTerminal( return { terminal, exact, status: 'unverifiable', reason: verdict.reason } } if (verdict?.status === 'live') { - return { terminal, exact, status: 'running' } + return { terminal, exact, status: 'live' } } return { terminal, exact, - status: terminal.connected === false ? 'exited' : 'running' + status: terminal.connected === false ? 'exited' : 'live' } } diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts b/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts index 4beb0323fd1..ecac17b8cbb 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-release-completion.ts @@ -10,6 +10,7 @@ import { type WorkerTerminalTailArchive } from '../../orchestration/worker-output-archive' import type { OrcaRuntimeService } from '../../orca-runtime' +import { describeUnconfirmedAgentStop } from '../../../../shared/pty-liveness-verdict' import { inspectWorkerTerminal } from './orchestration-worker-observation' import { orchestrationTimestampToMs } from './orchestration-worker-output' @@ -212,7 +213,19 @@ async function completeWorkerTerminalReleaseOnce( } try { - await runtime.closeTerminal(resource.terminal_handle) + const close = await runtime.closeTerminal(resource.terminal_handle) + if (!close.ptyKilled) { + const reason = describeUnconfirmedAgentStop(close) + const unknown = db.markWorkerTerminalReleaseUnknown(resource.id, reason) + return { + dispatchId, + state: 'release_unknown', + processAction: 'closed_agent_terminal', + archive: { source: archiveSource, status: archiveStatus }, + lastError: unknown.release_error ?? reason, + recovery: `Inspect with: orca orchestration worker-show --dispatch ${dispatchId} --json — then repeat worker-release with the same --retry-request. Never substitute a broad terminal close.` + } + } } catch (error) { const reason = error instanceof Error ? error.message : String(error) if (/disposed|not connected|unavailable/i.test(reason)) { diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts new file mode 100644 index 00000000000..6124b8b6cae --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-release-liveness-verdict.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from 'vitest' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' +import type { WorkerTerminalResourceRow } from '../../orchestration/worker-terminal-ownership' +import { completeWorkerTerminalRelease } from './orchestration-worker-release-completion' + +describe('orchestration worker release liveness verdict', () => { + it.each([ + { + name: 'an explicit unverifiable verdict', + close: { + handle: 'term_worker', + tabId: 'tab-worker', + ptyKilled: false, + ptyStopVerdict: 'unverifiable' as const, + ptyStopReason: 'its SSH provider is no longer registered' + }, + detail: 'its SSH provider is no longer registered' + }, + { + name: 'a bare unconfirmed close', + close: { handle: 'term_worker', tabId: 'tab-worker', ptyKilled: false }, + detail: 'the stop outcome could not be verified' + } + ])('does not release a worker after $name', async ({ close, detail }) => { + const reason = 'its SSH provider is no longer registered' + const resource = { + id: 'resource-1', + terminal_handle: 'term_worker', + host_scope: JSON.stringify({ kind: 'ssh', targetId: 'target-1' }), + archive_source: 'terminal', + archive_status: 'captured', + ownership_state: 'owned', + release_state: 'requested' + } as WorkerTerminalResourceRow + const runtime = { + showTerminal: vi.fn(async () => ({ handle: 'term_worker', connected: false })), + getTerminalPaneKey: vi.fn(() => 'tab-worker:leaf-worker'), + getTerminalProcessIncarnation: vi.fn(() => 'pty-worker:incarnation-1'), + getTerminalLivenessVerdict: vi.fn(() => ({ status: 'unverifiable', reason })), + getOrchestrationDispatchAuthority: vi.fn(() => ({ + hostScope: { kind: 'ssh', targetId: 'target-1' } + })), + closeTerminal: vi.fn(async () => close), + notifyMessageArrived: vi.fn() + } as unknown as OrcaRuntimeService + const markWorkerTerminalReleaseUnknown = vi.fn((_resourceId: string, releaseError: string) => ({ + ...resource, + release_state: 'unknown', + release_error: releaseError + })) + const db = { + getWorkerDispatch: vi.fn(() => ({ + agent_terminal_handle: 'term_worker', + created_at: '2026-08-16T00:00:00.000Z' + })), + isDispatchProcessCurrent: vi.fn(() => true), + workerTerminalResourceHasIdentityConflict: vi.fn(() => false), + getWorkerTerminalArchive: vi.fn(() => ({ kind: 'transcript_pin' })), + commitWorkerTerminalArchiveForRelease: vi.fn(() => ({ + ...resource, + release_state: 'releasing' + })), + markWorkerTerminalReleaseUnknown + } as unknown as OrchestrationDb + + await expect( + completeWorkerTerminalRelease({ + runtime, + db, + dispatchId: 'ctx-worker', + resource + }) + ).resolves.toMatchObject({ + state: 'release_unknown', + processAction: 'closed_agent_terminal', + lastError: `The agent terminal was closed but its process could not be confirmed stopped: ${detail}.` + }) + expect(markWorkerTerminalReleaseUnknown).toHaveBeenCalledWith( + 'resource-1', + `The agent terminal was closed but its process could not be confirmed stopped: ${detail}.` + ) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts index b5ba5b0000a..352022b4013 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-release.test.ts @@ -93,7 +93,8 @@ describe('orchestration worker release', () => { }) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_worker', - closed: true + tabId: 'tab-worker', + ptyKilled: true } as never) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) activeRunId = db.createRun({ diff --git a/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts index c500c2be9cc..1bf7e59d859 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-stop-liveness-verdict.test.ts @@ -98,6 +98,49 @@ describe('worker-stop against a terminal we lost contact with', () => { expect(stopped.lastError).not.toContain('exited') }) + it('does not settle a bare false close as stopped', async () => { + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: 'term_worker', + tabId: 'tab_worker', + ptyKilled: false + }) + const dispatch = createWorker() + + const stopped = (await call('orchestration.workerStop', { dispatch: dispatch.id })) as { + state: string + lastError: string + } + + expect(stopped.state).toBe('stop_unknown') + expect(stopped.lastError).toContain('could not be confirmed stopped') + }) + + it('uses the canonical live verdict for an observed running process', async () => { + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue({ + status: 'live', + ptyIds: ['runtime:pty:1'] + }) + const dispatch = createWorker() + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + observation: { status: 'live', exactWorker: true } + }) + }) + it('still reports a locally observed exit as exited', async () => { const dispatch = createWorker() diff --git a/src/main/runtime/rpc/methods/orchestration-worker-stop.ts b/src/main/runtime/rpc/methods/orchestration-worker-stop.ts index 119c4c7c4ad..15484db4d72 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-stop.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-stop.ts @@ -106,7 +106,7 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [ // the outcome honestly, never a reason to stop trying to stop the worker. if ( !observation.exact || - (observation.status !== 'running' && observation.status !== 'unverifiable') + (observation.status !== 'live' && observation.status !== 'unverifiable') ) { return unknownReceipt( params.dispatch, @@ -119,7 +119,7 @@ export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [ } try { const close = await runtime.closeTerminal(handle) - if (close.ptyStopVerdict) { + if (!close.ptyKilled) { // The tab is retired, but the agent process was never confirmed stopped — // settling here is the false success this receipt exists to prevent. return unknownReceipt( diff --git a/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts b/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts index 880620eee43..3f4679353bf 100644 --- a/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts @@ -30,7 +30,8 @@ describe('orchestration worker recovery', () => { }) vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ handle: 'term_worker', - closed: true + tabId: 'tab-worker', + ptyKilled: true } as never) }) @@ -80,7 +81,7 @@ describe('orchestration worker recovery', () => { call('orchestration.workerShow', { dispatch: dispatch.id }) ).resolves.toMatchObject({ worker: { state: 'ready' }, - observation: { status: 'running', exactWorker: true }, + observation: { status: 'live', exactWorker: true }, terminal: { handle: 'term_worker' } }) await expect( diff --git a/src/main/runtime/unstopped-pty-verification.ts b/src/main/runtime/unstopped-pty-verification.ts index 50ecfbb7a4b..dd013842a55 100644 --- a/src/main/runtime/unstopped-pty-verification.ts +++ b/src/main/runtime/unstopped-pty-verification.ts @@ -73,6 +73,20 @@ export function unverifiableStopVerdict( return null } +export async function resolveUnstoppedPtyVerdict( + failedPtyIds: readonly string[], + provider: IPtyProvider, + sweepBudgetMs: number, + providerObservesOwningHost: boolean, + runtime?: OrcaRuntimeService +): Promise { + if (failedPtyIds.length === 0) { + return { status: 'exited' } + } + const cached = providerObservesOwningHost ? null : unverifiableStopVerdict(failedPtyIds, runtime) + return cached ?? verifyUnstoppedPtys(failedPtyIds, provider, sweepBudgetMs) +} + /** Names the blocking PTYs so a wedged removal is diagnosable, not just refused. */ export function describeUnstoppedPtys( worktreeId: string, diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index b51f292c29c..ed29bc26e2f 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -521,7 +521,7 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.shutdown).toHaveBeenCalledTimes(1) }) - it('accepts a failed Windows stop when a fresh inventory proves the PTY exited', async () => { + it('accepts a failed stop when exact-owner inventory supersedes cached uncertainty', async () => { const worktreeId = 'repo-1::C:/Users/User/orca/workspaces/repo/feature' const ptyId = `${worktreeId}@@windows-pty` const stopTerminalsForWorktree = vi.fn( @@ -538,7 +538,11 @@ describe('killAllProcessesForWorktree', () => { }) ) const runtime = { - stopTerminalsForWorktree + stopTerminalsForWorktree, + getPtyLivenessVerdict: vi.fn(() => ({ + status: 'unverifiable', + reason: 'the provider disconnected during stop' + })) } as unknown as Parameters[1]['runtime'] let inventoryCount = 0 const localProvider = createProviderStub(async () => { @@ -568,6 +572,45 @@ describe('killAllProcessesForWorktree', () => { expect(localProvider.listProcesses).toHaveBeenCalledTimes(2) }) + it('keeps cached uncertainty when no provider can observe the owning host', async () => { + const worktreeId = 'repo-1::/remote/worktree' + const ptyId = `${worktreeId}@@remote-pty` + const stopTerminalsForWorktree = vi.fn( + async ( + _worktreeId: string, + options: { + stopPty: ( + ptyId: string, + stop: () => boolean + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => ({ + stopped: (await options.stopPty(ptyId, () => false)).owner ? 1 : 0 + }) + ) + const runtime = { + stopTerminalsForWorktree, + getPtyLivenessVerdict: vi.fn(() => ({ + status: 'unverifiable', + reason: 'no registered provider can observe its host' + })) + } as unknown as Parameters[1]['runtime'] + const fallbackProvider = createProviderStub(async () => []) + listRegisteredPtysMock.mockReturnValue([]) + + await expect( + killAllProcessesForWorktree(worktreeId, { + runtime, + resolvedConnectionId: 'missing-connection', + localProvider: fallbackProvider, + includeProviderInventory: false, + includeLocalRegistry: false, + requirePhysicalStop: true + }) + ).rejects.toThrow('no registered provider can observe its host') + expect(fallbackProvider.listProcesses).not.toHaveBeenCalled() + }) + it('keeps duplicate sweeps behind the runtime physical-stop promise', async () => { let releasePhysicalStop: () => void = () => undefined const physicalStop = new Promise((resolve) => { diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 1f318685d29..a28875dff24 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -15,9 +15,7 @@ import { describeError, describeFailedPtySweep, describeUnstoppedPtys, - unverifiableStopVerdict, - verifyUnstoppedPtys, - type UnstoppedPtyVerdict + resolveUnstoppedPtyVerdict } from './unstopped-pty-verification' // Why: normal inventories still coalesce into one process scan, while a stale @@ -232,11 +230,15 @@ export async function killAllProcessesForWorktree( [...stopAttempts].map(async ([ptyId, stopped]) => [ptyId, await stopped] as const) ) const failedPtyIds = stopResults.filter(([, stopped]) => !stopped).map(([ptyId]) => ptyId) - const verdict: UnstoppedPtyVerdict = - failedPtyIds.length === 0 - ? { status: 'exited' } - : (unverifiableStopVerdict(failedPtyIds, deps.runtime) ?? - (await verifyUnstoppedPtys(failedPtyIds, deps.localProvider, sweepBudgetMs))) + const verdict = await resolveUnstoppedPtyVerdict( + failedPtyIds, + deps.localProvider, + sweepBudgetMs, + deps.includeProviderInventory !== false || + (deps.resolvedConnectionId === undefined && + deps.resolvedRuntimeEnvironmentId === undefined), + deps.runtime + ) if (verdict.status === 'exited') { for (const ptyId of failedPtyIds) { clearStoppedPtyState(ptyId, deps.onPtyStopped) @@ -365,13 +367,10 @@ async function sweepRegistryForWorktree( } function clearStoppedPtyState(ptyId: string, onPtyStopped?: (ptyId: string) => void): void { - if (!onPtyStopped) { - return - } try { // Why: daemon shutdown does not always fan a local pty:exit event back // through pty.ts, but removed worktrees must immediately drop memory rows. - onPtyStopped(ptyId) + onPtyStopped?.(ptyId) } catch { /* cleanup is best-effort and must not block git-level removal */ } diff --git a/src/shared/pty-liveness-verdict.ts b/src/shared/pty-liveness-verdict.ts index cb27de57a0f..13d609f13f0 100644 --- a/src/shared/pty-liveness-verdict.ts +++ b/src/shared/pty-liveness-verdict.ts @@ -28,6 +28,6 @@ export function describeUnconfirmedAgentStop(close: { const detail = close.ptyStopVerdict === 'live' ? 'it is still running' - : (close.ptyStopReason ?? 'its host could not be reached') + : (close.ptyStopReason ?? 'the stop outcome could not be verified') return `The agent terminal was closed but its process could not be confirmed stopped: ${detail}.` }