diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 7e8bc5b3b7e..d462aa996de 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -14330,22 +14330,24 @@ "platforms": ["macos", "linux", "windows"], "providers": ["local-daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": ["macos", "linux"], - "coveredProviders": ["local-daemon", "ssh"], - "coverageNotes": "Deterministic production-boundary tests cover persisted daemon owner identity, full-owner retirement CAS and rollback, local daemon routing, pre-side-effect daemon attach fencing, SSH provider/relay incarnation checks, exact relay exit proof, renderer preservation, and guarded recovery. A real macOS multi-generation daemon journey and rendered Docker SSH reconnect journey passed. Paired headed/headless was blocked by missing web-client URL; targeted A/B update handoff, WSL, ConPTY, and physical Windows remain gaps.", + "coveredProviders": ["local-daemon", "ssh", "remote-runtime"], + "coverageNotes": "Deterministic production-boundary tests cover persisted daemon owner identity, full-owner retirement CAS and rollback, local daemon routing, pre-side-effect daemon attach fencing, SSH provider/relay incarnation checks, exact relay exit proof, renderer preservation, remote-runtime failure classification, and guarded recovery. Real macOS multi-generation, rendered Docker SSH reconnect, and paired remote materialization/reconnect journeys passed. The headed/headless probe-gap journey was blocked by missing web-client URL; targeted A/B update handoff, WSL, ConPTY, and physical Windows remain gaps.", "motivatingLinks": [ "https://linear.app/stably/issue/STA-5269", "https://github.com/stablyai/orca/pull/16152" ], "invariant": "Only the exact execution host, owner incarnation, and session incarnation may report live or exited. Missing routes, adapters, metadata, or contact are unverifiable and preserve every PTY binding, terminal row, pane, tab, and retry lease.", - "oracle": "Persist a PTY owned by daemon A, replace the compatible current slot with daemon B, and have B report the logical id absent. Require an unverifiable verdict without querying B as the owner, without broad fallback, deletion, fresh spawn, or unrelated stop. Exact A plus the exact session may report live or exited; a mismatched reused incarnation, provider loss, and owner-unverified result or throw preserve topology and settle guarded recovery.", + "oracle": "Persist a PTY owned by daemon A, replace the compatible current slot with daemon B, and have B report the logical id absent. Require an unverifiable verdict without querying B as the owner, without broad fallback, deletion, fresh spawn, or unrelated stop. Exact A plus the exact session may report live or exited; an intentional history-preserving stop retains that exact authority and may restore the pane. A mismatched reused incarnation, provider loss, and owner-unverified result or throw preserve topology and settle guarded recovery.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-authority.oracle.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-resolution.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/daemon-pty-router-owner-fencing.test.ts src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts src/main/daemon/daemon-pty-adapter-session-adoption.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-incarnation-fence.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts src/main/daemon/daemon-pty-adapter-history-recovery.test.ts src/main/daemon/daemon-pty-history-stop-inventory.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-startup-barrier-and-listing.test.ts src/main/ipc/pty-persisted-incarnation-repair.test.ts src/main/ipc/pty-dead-owner-respawn.test.ts src/main/ipc/pty-controller-spawn-admission.test.ts src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty/pane/stable-owner-retirement.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-spawn.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts src/relay/pty-handler-attach-replay.test.ts --reporter=dot", - "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts --reporter=dot", "pnpm exec playwright test tests/e2e/daemon-generation-reconnect-safety.spec.ts tests/e2e/daemon-generation-legacy-close-safety.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1", + "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1", "pnpm exec playwright test tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1" ], @@ -14356,6 +14358,9 @@ "src/main/daemon/daemon-pty-router-owner-fencing.test.ts", "src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts", "src/main/daemon/daemon-pty-adapter-session-adoption.test.ts", + "src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts", + "src/main/daemon/daemon-pty-adapter-history-recovery.test.ts", + "src/main/daemon/daemon-pty-history-stop-inventory.test.ts", "src/main/daemon/terminal-host.test.ts", "src/main/daemon/terminal-host-incarnation-fence.test.ts", "src/main/daemon/degraded-daemon-pty-provider.test.ts", @@ -14374,8 +14379,10 @@ "src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts", "src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts", "src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts", + "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts", "tests/e2e/daemon-generation-reconnect-safety.spec.ts", "tests/e2e/daemon-generation-legacy-close-safety.spec.ts", + "tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts", "tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts", "tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts", "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts" @@ -14388,6 +14395,24 @@ "only an exact routed owner and session can confirm absence" ] }, + { + "file": "src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts", + "assertions": [ + "a history-preserving stop retains exact owner authority and reports that session exited" + ] + }, + { + "file": "src/main/daemon/daemon-pty-adapter-history-recovery.test.ts", + "assertions": [ + "a delayed physical exit after an unreadable history-preserving stop cannot erase exact exit authority" + ] + }, + { + "file": "src/main/daemon/daemon-pty-history-stop-inventory.test.ts", + "assertions": [ + "authenticated inventory racing a history-preserving exit cannot erase exact exit authority" + ] + }, { "file": "src/main/daemon/terminal-host-incarnation-fence.test.ts", "assertions": [ @@ -14412,6 +14437,13 @@ "assertions": [ "result and thrown owner-unverified outcomes settle the exact retry and preserve topology" ] + }, + { + "file": "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts", + "assertions": [ + "runtime outages remain quiet reconnect state while only explicit owner-unverified verdicts show ownership recovery UI", + "generic saved-pane failures retain their original diagnostic without fabricating an owner verdict" + ] } ], "evidenceRuns": [ @@ -14479,13 +14511,13 @@ "summary": "Three SSH provider and relay files passed 54 attach, incarnation, source-recovery, and exact-exit tests." }, { - "date": "2026-08-23", + "date": "2026-08-25", "runner": "local", "platform": "macos", "result": "passed", - "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts --reporter=dot", - "durationSeconds": 4.07, - "summary": "Six renderer files passed 65 liveness, typed-result, topology-preservation, and bounded recovery tests." + "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts --reporter=dot", + "durationSeconds": 3.72, + "summary": "Seven renderer files passed 85 liveness, failure-classification, topology-preservation, and bounded recovery tests." }, { "date": "2026-08-25", diff --git a/src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts b/src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts index 3bdd53034d6..8c45978d5b2 100644 --- a/src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts +++ b/src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { join } from 'node:path' import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { DaemonPtyAdapter } from './daemon-pty-adapter' +import { TerminalSessionExitedError } from './daemon-errors' import type { DaemonServer } from './daemon-server' import { getHistorySessionDirName } from './history-paths' import type { PendingOutputRecord } from './types' @@ -715,12 +716,13 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { it('writes a final checkpoint before keepHistory shutdown', async () => { historyAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, historyPath: historyDir }) - const { id } = await historyAdapter.spawn({ + const spawned = await historyAdapter.spawn({ cols: 80, rows: 24, cwd: '/home/user', sessionId: 'sleep-checkpoint' }) + const { id } = spawned const checkpointSpy = vi.spyOn(historyAdapter.getHistoryManager()!, 'checkpoint') lastSubprocess._simulateData('fresh output before sleep\r\n') @@ -732,6 +734,21 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { { pendingOutputSeq: expect.any(Number) } ) expect(existsSync(join(historyDir, getHistorySessionDirName(id)))).toBe(true) + expect(historyAdapter.getTerminalOwnerIdentity(id)).toEqual(spawned.ownerIdentity) + await expect( + historyAdapter.probePtyLiveness(id, spawned.incarnationId, spawned.ownerIdentity) + ).resolves.toBe(false) + await expect( + historyAdapter.spawn({ + sessionId: id, + attachOnly: true, + expectedIncarnationId: spawned.incarnationId, + expectedIncarnationIsAuthoritative: true, + expectedOwnerIdentity: spawned.ownerIdentity, + cols: 80, + rows: 24 + }) + ).rejects.toBeInstanceOf(TerminalSessionExitedError) }) itOnPosix('persists final take records that are not represented in the snapshot', async () => { diff --git a/src/main/daemon/daemon-pty-adapter-history-recovery.test.ts b/src/main/daemon/daemon-pty-adapter-history-recovery.test.ts index 820f318d6a8..41898f9c452 100644 --- a/src/main/daemon/daemon-pty-adapter-history-recovery.test.ts +++ b/src/main/daemon/daemon-pty-adapter-history-recovery.test.ts @@ -146,6 +146,34 @@ describe('DaemonPtyAdapter history recovery', () => { expect(JSON.parse(readFileSync(metaPath, 'utf-8')).endedAt).toBeNull() }) + it('retains exact exit authority when an unreadable keepHistory stop exits late', async () => { + historyAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, historyPath: historyDir }) + const spawned = await historyAdapter.spawn({ + cols: 80, + rows: 24, + sessionId: 'sleep-unreadable-late-exit' + }) + const reader = (historyAdapter as unknown as { historyReader: HistoryReader }).historyReader + vi.spyOn(reader, 'detectColdRestoreState').mockResolvedValue({ + status: 'unreadable', + sessionId: spawned.id + }) + const exited = new Promise((resolve) => { + historyAdapter.onExit((event) => { + if (event.id === spawned.id) { + resolve() + } + }) + }) + + await historyAdapter.shutdown(spawned.id, { immediate: true, keepHistory: true }) + await exited + + await expect( + historyAdapter.probePtyLiveness(spawned.id, spawned.incarnationId, spawned.ownerIdentity) + ).resolves.toBe(false) + }) + it('suspends an empty final checkpoint with unreadable post-checkpoint recovery', async () => { historyAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, historyPath: historyDir }) const { id } = await historyAdapter.spawn({ diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index 292ab175f43..e3479ab584b 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -99,7 +99,10 @@ export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyPro this.pausedProducerSessionIds.delete(event.sessionId) this.producerResumesOwedOnReconnect.delete(event.sessionId) this.backgroundedSessionIds.delete(event.sessionId) - if (!this.sleepRestoreSessionIds.has(event.sessionId)) { + const preservesRestorableAuthority = + this.sleepRestoreSessionIds.has(event.sessionId) || + this.historyPreservingStopSessionIds.has(event.sessionId) + if (!preservesRestorableAuthority) { this.coldRestoreCache.delete(event.sessionId) } this.sessionsNeedingFullCheckpoint.delete(event.sessionId) @@ -117,8 +120,10 @@ export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyPro } this.initialCwds.delete(event.sessionId) this.wslDistrosBySessionId.delete(event.sessionId) - this.sessionIncarnations.delete(event.sessionId) - this.sessionOwnerIdentities.delete(event.sessionId) + if (!preservesRestorableAuthority) { + this.sessionIncarnations.delete(event.sessionId) + this.sessionOwnerIdentities.delete(event.sessionId) + } // oxlint-disable-next-line unicorn/no-useless-spread -- copy-safe: listeners may unsubscribe during iteration for (const listener of [...this.exitListeners]) { listener({ diff --git a/src/main/daemon/daemon-pty-event-subscriptions.ts b/src/main/daemon/daemon-pty-event-subscriptions.ts index 20929def1ba..395dbca98af 100644 --- a/src/main/daemon/daemon-pty-event-subscriptions.ts +++ b/src/main/daemon/daemon-pty-event-subscriptions.ts @@ -79,6 +79,7 @@ export abstract class DaemonPtyEventSubscriptions extends DaemonPtySessionInvent this.overlayDeadlineWarnedSessionIds.clear() this.periodicDeadlineWarnedSessionIds.clear() this.nonFinalAdmissionDeniedSessionIds.clear() + this.historyPreservingStopSessionIds.clear() this.coldRestoreCache.clear() this.wslDistrosBySessionId.clear() this.pausedProducerSessionIds.clear() diff --git a/src/main/daemon/daemon-pty-history-stop-inventory.test.ts b/src/main/daemon/daemon-pty-history-stop-inventory.test.ts new file mode 100644 index 00000000000..f7fda6a1da7 --- /dev/null +++ b/src/main/daemon/daemon-pty-history-stop-inventory.test.ts @@ -0,0 +1,47 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { rmSync } from 'node:fs' +import type { DaemonPtyAdapter } from './daemon-pty-adapter' +import type { DaemonServer } from './daemon-server' +import { createMockSubprocess, startDaemonAdapterHarness } from './daemon-pty-adapter-test-harness' + +describe('DaemonPtyAdapter history-stop inventory', () => { + let dir: string + let server: DaemonServer + let adapter: DaemonPtyAdapter + let lastSubprocess: ReturnType + + beforeEach(async () => { + const harness = await startDaemonAdapterHarness(() => { + lastSubprocess = createMockSubprocess() + return lastSubprocess + }) + dir = harness.dir + server = harness.server + adapter = harness.adapter + }) + + afterEach(async () => { + adapter.dispose() + await server.shutdown() + rmSync(dir, { recursive: true, force: true }) + }) + + it('retains exact authority when inventory overtakes a history-preserving exit', async () => { + const spawned = await adapter.spawn({ cols: 80, rows: 24 }) + const internals = adapter as unknown as { + activeSessionIds: Set + historyPreservingStopSessionIds: Set + } + internals.historyPreservingStopSessionIds.add(spawned.id) + lastSubprocess._simulateExit(0) + internals.activeSessionIds.add(spawned.id) + + await adapter.listProcesses() + + expect(adapter.hasPty(spawned.id)).toBe(false) + expect(adapter.getTerminalOwnerIdentity(spawned.id)).toEqual(spawned.ownerIdentity) + await expect( + adapter.probePtyLiveness(spawned.id, spawned.incarnationId, spawned.ownerIdentity) + ).resolves.toBe(false) + }) +}) diff --git a/src/main/daemon/daemon-pty-router-owner-fencing.test.ts b/src/main/daemon/daemon-pty-router-owner-fencing.test.ts index 3cd6909542e..eb36b9661a4 100644 --- a/src/main/daemon/daemon-pty-router-owner-fencing.test.ts +++ b/src/main/daemon/daemon-pty-router-owner-fencing.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it, vi } from 'vitest' import type { DaemonPtyAdapter } from './daemon-pty-adapter' import { DaemonPtyRouter } from './daemon-pty-router' import type { PtySpawnOptions, PtySpawnResult } from '../providers/types' +import type { TerminalOwnerIdentity } from '../../shared/terminal-owner-identity' +import { PROTOCOL_VERSION } from './daemon-protocol-version' type FenceAdapter = DaemonPtyAdapter & { emitExit: (id: string, code: number, incarnationId: string) => void @@ -23,6 +25,7 @@ function createFenceAdapter(label: string, initialSessions: string[] = []): Fenc listProcesses: vi.fn(async () => sessions.map((id) => ({ id, incarnationId: `${label}:${id}`, cwd: '', title: label })) ), + probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)), write: vi.fn(), shutdown: vi.fn(), onData: noopSubscription, @@ -46,6 +49,34 @@ function createFenceAdapter(label: string, initialSessions: string[] = []): Fenc } describe('DaemonPtyRouter owner fencing', () => { + it('retains the exact owner route for a history-preserving stop', async () => { + const current = createFenceAdapter('current') + const legacy = createFenceAdapter('legacy', ['parked-session']) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + await router.discoverLegacySessions() + const parkedOwner = { + executionHostId: 'local', + ownerKind: 'daemon', + ownerIncarnationId: 'legacy-daemon', + sessionIncarnationId: 'legacy:parked-session', + protocolVersion: PROTOCOL_VERSION, + endpointRef: 'daemon-v37' + } satisfies TerminalOwnerIdentity + legacy.getTerminalOwnerIdentity = vi.fn(() => parkedOwner) + legacy.probePtyLiveness = vi.fn(async () => false) + vi.mocked(legacy.shutdown).mockImplementation(async () => { + legacy.emitExit('parked-session', -1, 'legacy:parked-session') + }) + + await router.shutdown('parked-session', { keepHistory: true }) + + await expect( + router.probePtyLiveness('parked-session', 'legacy:parked-session', parkedOwner) + ).resolves.toBe(false) + expect(legacy.probePtyLiveness).toHaveBeenCalledOnce() + expect(current.probePtyLiveness).not.toHaveBeenCalled() + }) + it('keeps replacement B routed after a delayed exit from owner A', async () => { const current = createFenceAdapter('current') const legacy = createFenceAdapter('legacy', ['reused-session']) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 1acbec4372e..a56939c6fa5 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -29,6 +29,13 @@ export class DaemonPtyRouter implements IPtyProvider { this.subscriptions = new DaemonPtyAdapterSubscriptionFanout( this.allAdapters(), (adapter, payload) => { + const retainedOwner = adapter.getTerminalOwnerIdentity?.(payload.id) + if ( + payload.incarnationId && + retainedOwner?.sessionIncarnationId === payload.incarnationId + ) { + return + } this.ownerResolver.forgetRoute(payload.id, adapter, payload.incarnationId) }, (adapter) => this.ownerResolver.invalidateProvider(adapter) diff --git a/src/main/daemon/daemon-pty-runtime-state.ts b/src/main/daemon/daemon-pty-runtime-state.ts index 897f744de82..11bc08326ab 100644 --- a/src/main/daemon/daemon-pty-runtime-state.ts +++ b/src/main/daemon/daemon-pty-runtime-state.ts @@ -149,6 +149,7 @@ export abstract class DaemonPtyRuntimeState { protected overlayDeadlineWarnedSessionIds = new Set() protected periodicDeadlineWarnedSessionIds = new Set() protected keepHistoryShutdowns = new Set>() + protected historyPreservingStopSessionIds = new Set() protected disconnectOnlyPromise: Promise | null = null protected supportsCheckpoints: boolean protected supportsIncrementalCheckpoints: boolean @@ -269,8 +270,12 @@ export abstract class DaemonPtyRuntimeState { if (!incarnationId || !this.lastAuthenticatedIdentity) { return } + const previousIncarnationId = this.sessionIncarnations.get(id) this.sessionIncarnations.set(id, incarnationId) this.sessionOwnerIdentities.set(id, { ...this.lastAuthenticatedIdentity }) + if (previousIncarnationId && previousIncarnationId !== incarnationId) { + this.historyPreservingStopSessionIds.delete(id) + } } protected hasExactSessionAuthority( diff --git a/src/main/daemon/daemon-pty-session-control.ts b/src/main/daemon/daemon-pty-session-control.ts index 6d0481ca5f0..12a46dce94b 100644 --- a/src/main/daemon/daemon-pty-session-control.ts +++ b/src/main/daemon/daemon-pty-session-control.ts @@ -168,6 +168,9 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority if (opts.keepHistory && this.disconnectOnlyPromise) { throw new Error('Cannot keep history after daemon disconnect has started') } + if (opts.keepHistory) { + this.historyPreservingStopSessionIds.add(id) + } const shutdown = this.withHistorySpawnLock(id, () => this.shutdownWithHistoryLock(id, opts)) if (!opts.keepHistory) { await shutdown @@ -176,6 +179,9 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority this.keepHistoryShutdowns.add(shutdown) try { await shutdown + } catch (error) { + this.historyPreservingStopSessionIds.delete(id) + throw error } finally { this.keepHistoryShutdowns.delete(shutdown) } @@ -242,8 +248,11 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority remainingDaemonRequestTimeoutMs(opts.deadlineMs) ) this.activeSessionIds.delete(id) - this.sessionIncarnations.delete(id) - this.sessionOwnerIdentities.delete(id) + if (!opts.keepHistory) { + this.historyPreservingStopSessionIds.delete(id) + this.sessionIncarnations.delete(id) + this.sessionOwnerIdentities.delete(id) + } this.clearSessionAwaitingDaemonRecovery(id) this.dirtySessionVersions.delete(id) if (!opts.keepHistory) { @@ -282,6 +291,7 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority } ackColdRestore(sessionId: string): void { + this.historyPreservingStopSessionIds.delete(sessionId) this.coldRestoreCache.delete(sessionId) this.sleepRestoreSessionIds.delete(sessionId) } diff --git a/src/main/daemon/daemon-pty-session-inventory.ts b/src/main/daemon/daemon-pty-session-inventory.ts index 2129121ed5a..55b8b53af1e 100644 --- a/src/main/daemon/daemon-pty-session-inventory.ts +++ b/src/main/daemon/daemon-pty-session-inventory.ts @@ -62,8 +62,10 @@ export abstract class DaemonPtySessionInventory extends DaemonPtyProcessInspecti for (const id of preRequestActiveIds) { if (!aliveSessionIds.has(id)) { this.activeSessionIds.delete(id) - this.sessionIncarnations.delete(id) - this.sessionOwnerIdentities.delete(id) + if (!this.historyPreservingStopSessionIds.has(id)) { + this.sessionIncarnations.delete(id) + this.sessionOwnerIdentities.delete(id) + } } } this.publishAuditObservation( @@ -140,6 +142,7 @@ export abstract class DaemonPtySessionInventory extends DaemonPtyProcessInspecti this.overlayDeadlineWarnedSessionIds.clear() this.periodicDeadlineWarnedSessionIds.clear() this.nonFinalAdmissionDeniedSessionIds.clear() + this.historyPreservingStopSessionIds.clear() this.pausedProducerSessionIds.clear() this.producerResumesOwedOnReconnect.clear() this.stopCheckpointTimer() diff --git a/src/main/daemon/daemon-pty-session-spawn.ts b/src/main/daemon/daemon-pty-session-spawn.ts index 2feb3281b40..1c8d3775de9 100644 --- a/src/main/daemon/daemon-pty-session-spawn.ts +++ b/src/main/daemon/daemon-pty-session-spawn.ts @@ -1,5 +1,7 @@ import { recognizeAgentProcessFromCommandLine } from '../../shared/agent-process-recognition' import { shouldUseShellReadyStartupDelivery } from '../../shared/codex-startup-delivery' +import { sameTerminalOwnerIdentity } from '../../shared/terminal-owner-identity' +import { SessionNotFoundError, TerminalSessionExitedError } from './daemon-errors' import type { HistoryRecoveryContext, PendingDaemonSpawnOperation @@ -49,6 +51,20 @@ export abstract class DaemonPtySessionSpawn extends DaemonPtySpawnResult { this.doSpawn({ ...spawnOpts, sessionId }, operation, historyRecovery) ) ) + } catch (error) { + const exactOwner = this.getTerminalOwnerIdentity(sessionId) + if ( + error instanceof SessionNotFoundError && + spawnOpts.attachOnly === true && + spawnOpts.expectedIncarnationIsAuthoritative === true && + spawnOpts.expectedIncarnationId !== undefined && + this.hasExactSessionAuthority(sessionId, spawnOpts.expectedIncarnationId) && + (spawnOpts.expectedOwnerIdentity === undefined || + sameTerminalOwnerIdentity(exactOwner, spawnOpts.expectedOwnerIdentity)) + ) { + throw new TerminalSessionExitedError(sessionId) + } + throw error } finally { if (historyRecovery.freeze) { this.historyManager?.abandonRecoveryFreeze(historyRecovery.freeze) diff --git a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts index da3d4ae6dd5..15e249bc31f 100644 --- a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts +++ b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts @@ -75,6 +75,24 @@ describe('SSH PTY provider session reattach incarnation', () => { ) }) + it('rejects a legacy attach that cannot echo a persisted incarnation', async () => { + const mux = { + request: vi.fn().mockResolvedValue({ replay: 'saved output' }), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.spawn({ + cols: 80, + rows: 24, + sessionId: 'pty-old', + expectedIncarnationId: 'incarnation-original' + }) + ).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) + }) + it('keeps relay absence owner-unverified', async () => { const mux = { request: vi.fn().mockRejectedValue(new Error('PTY "pty-old" not found')), diff --git a/src/renderer/src/components/terminal-pane/TerminalErrorToast.tsx b/src/renderer/src/components/terminal-pane/TerminalErrorToast.tsx index b56cb7b5fdd..d1beeafd443 100644 --- a/src/renderer/src/components/terminal-pane/TerminalErrorToast.tsx +++ b/src/renderer/src/components/terminal-pane/TerminalErrorToast.tsx @@ -191,7 +191,7 @@ export function TerminalErrorToast({
{translate( 'auto.components.terminal.pane.TerminalErrorToast.ownerUnavailableTitle', - 'Terminal session unavailable' + 'Terminal session not connected' )}
@@ -201,7 +201,7 @@ export function TerminalErrorToast({
{translate( 'auto.components.terminal.pane.TerminalErrorToast.ownerUnavailableDetail', - "Orca couldn't confirm whether the previous session is still running. The tab and session are preserved; retry when the owner is available." + "Orca couldn't verify whether this saved session is still running, so it left the tab and history untouched. Check again retries the saved panes in this tab; it won't close or replace them." )}
{footer ? ( @@ -213,7 +213,7 @@ export function TerminalErrorToast({
diff --git a/src/renderer/src/components/terminal-pane/TerminalPane.tsx b/src/renderer/src/components/terminal-pane/TerminalPane.tsx index 2aaa5954d08..1769bdef145 100644 --- a/src/renderer/src/components/terminal-pane/TerminalPane.tsx +++ b/src/renderer/src/components/terminal-pane/TerminalPane.tsx @@ -64,7 +64,6 @@ import { stripSshReconnectOwnedErrorLines, TerminalErrorToast } from './TerminalErrorToast' -import { TERMINAL_PANE_OWNER_UNVERIFIED } from '../../../../shared/terminal-pane-owner-verdict' import { TerminalProcessExitOverlay } from './TerminalProcessExitOverlay' import { TerminalSessionStateSaveFailureDialog } from './TerminalSessionStateSaveFailureDialog' import TerminalContextMenu from './TerminalContextMenu' @@ -507,8 +506,9 @@ function TerminalPane( } }, []) const retryTerminalConnection = useCallback(() => { - setTerminalError(null) - useAppStore.getState().remountTerminalTabForRecovery(tabId) + if (useAppStore.getState().remountTerminalTabForRecovery(tabId)) { + setTerminalError(null) + } }, [tabId]) const onPtyRecoveryStateRef = useRef( (paneId: number, state: PtyTransportRecoveryState | null) => { @@ -2932,32 +2932,6 @@ function TerminalPane( const activePane = managerRef.current?.getActivePane() const managedPanes = managerRef.current?.getPanes() ?? [] - // A daemon handoff can leave the renderer waiting on a preserved PTY without - // producing an IPC error. Surface the same explicit retry state instead of - // leaving the user with an unexplained blank pane. - useEffect(() => { - const currentPanes = managerRef.current?.getPanes() ?? [] - if (!isActive || !isVisible || currentPanes.length === 0) { - return - } - const savedPtyIds = Object.values(savedLayout.ptyIdsByLeafId ?? {}).filter(Boolean) - if (savedPtyIds.length === 0) { - return - } - const hasUnboundPane = currentPanes.some( - (pane) => paneTransportsRef.current.get(pane.id)?.getPtyId() == null - ) - if (!hasUnboundPane) { - if (terminalError && isPaneOwnerUnverifiedError(terminalError)) { - setTerminalError(null) - } - return - } - const timer = window.setTimeout(() => { - setTerminalError((current) => current ?? TERMINAL_PANE_OWNER_UNVERIFIED) - }, 1_500) - return () => window.clearTimeout(timer) - }, [isActive, isVisible, managerRef, paneCount, paneLayoutRevision, savedLayout, terminalError]) const showSshReconnectOverlay = Boolean( isActive && isVisible && diff --git a/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts index 5759b6298bd..7ae3567f665 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts @@ -403,6 +403,7 @@ describe('connectPanePty', () => { expect(deps.clearTabPtyId).not.toHaveBeenCalled() expect(deps.syncPanePtyLayoutBinding).not.toHaveBeenCalled() expect(deps.updateTabPtyId).not.toHaveBeenCalled() + expect(deps.onPtyErrorRef.current).not.toHaveBeenCalled() }) it.each([ @@ -474,8 +475,39 @@ describe('connectPanePty', () => { { id: 'tab-1', ptyId: null, generation: 7 } ]) expect(mockStoreState.deferredSshSessionIdsByTabId['tab-1']).toBe(restoredPtyId) + if (outcome === 'reject') { + expect(deps.onPtyErrorRef.current).toHaveBeenCalledWith(1, 'relay attach timed out') + } else { + expect(deps.onPtyErrorRef.current).not.toHaveBeenCalled() + } + expect(deps.onPtyErrorRef.current).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('terminal_pane_owner_unverified') + ) } ) + + it('surfaces a generic restored-pane failure without calling it owner-unverified', async () => { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport() + transport.connect.mockRejectedValueOnce(new Error('daemon package is unavailable')) + transportFactoryQueue.push(transport) + const deps = createDeps({ + restoredLeafId: LEAF_2, + restoredPtyIdByLeafId: { [LEAF_2]: 'preserved-pty' } + }) + + connectPanePty(createPane(2) as never, createManager(2) as never, deps as never) + await flushAsyncTicks() + + expect(deps.onPtyErrorRef.current).toHaveBeenCalledWith(2, 'daemon package is unavailable') + expect(deps.onPtyErrorRef.current).not.toHaveBeenCalledWith( + expect.anything(), + expect.stringContaining('terminal_pane_owner_unverified') + ) + expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + }) it.each(['result', 'throw'] as const)( 'preserves a restored pane binding when its daemon owner is unverifiable (%s)', async (outcome) => { diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts index 1e343280ebe..26ac0cfdfaa 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts @@ -15,6 +15,7 @@ import type { ConnectPanePtySession } from './connect-pane-pty-session' import { runDeferredSessionReattachChoice } from './deferred-session-reattach-choice' import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery' +import { isTerminalPaneOwnerUnverified } from '../../../../../shared/terminal-pane-owner-verdict' export function runDeferredSessionAttach(session: ConnectPanePtySession): void { // Why: trigger the deferred SSH connect per-tab (not per-target) so multiple tabs for one target reattach independently. @@ -291,8 +292,13 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void { }) return } - session.reportError(err instanceof Error ? err.message : String(err)) - recoverUnverifiableReattach(session, pendingSessionId) + const message = err instanceof Error ? err.message : String(err) + session.reportError(message) + if (isTerminalPaneOwnerUnverified(message)) { + recoverUnverifiableReattach(session, pendingSessionId) + } else if (session.directSshRetryAttempt) { + session.settleDirectSshPaneRetryAttempt(session.directSshRetryAttempt, 'failed') + } }) session.armDirectSshPaneRetryTimeout( trackedReattachPromise, diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts index eb87d12fb6d..efb5603db00 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts @@ -165,7 +165,9 @@ export function startDeferredSessionReattach( return } session.reportError(message) - recoverUnverifiableReattach(session, deferredReattachSessionId) + if (session.directSshRetryAttempt) { + session.settleDirectSshPaneRetryAttempt(session.directSshRetryAttempt, 'failed') + } }) session.armDirectSshPaneRetryTimeout(trackedReattachPromise, session.directSshRetryAttempt) } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts b/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts index 74951c4c0bc..de5588cee6a 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts @@ -42,6 +42,7 @@ type ReattachResultSession = ReattachPayloadSession & | 'scheduleReattachIdleAgentCursorReset' | 'serializeHiddenOutputSnapshot' | 'setPanePtyFitBinding' + | 'settleDirectSshPaneRetryAttempt' | 'startFreshColdRestoreAgentResume' | 'structuralReplayCoordinator' | 'syncHiddenRendererPtyDelivery' @@ -95,7 +96,9 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi paneId: session.pane.id, ptyId: staleSessionId ?? null }) - recoverUnverifiableReattach(sessionBag, staleSessionId) + if (session.directSshRetryAttempt) { + session.settleDirectSshPaneRetryAttempt(session.directSshRetryAttempt, 'failed') + } return false } session.registerEffectiveLaunchConfig(connectResult?.launchConfig, { diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts index 283abb2cec4..2237ebf61e1 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts @@ -96,6 +96,75 @@ describe('createRemoteRuntimePtyTransport', () => { expect(latestSubscribePayload()).toMatchObject({ terminal: 'hub-terminal-1' }) }) + it('keeps a saved host pane in reconnect recovery for a runtime outage', async () => { + runtimeCall.mockRejectedValue( + Object.assign(new Error('Remote Orca runtime closed the connection.'), { + code: 'remote_runtime_unavailable' + }) + ) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const transport = createRemoteRuntimePtyTransport('hub-env', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + + const result = await transport.connect({ + url: '', + sessionId: 'ssh:hub-private@@pty-2', + callbacks: { onError } + }) + + expect(result).toBeUndefined() + expect(onError).not.toHaveBeenCalled() + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + }) + + it('returns owner-unverifiable only for an explicit saved-pane owner verdict', async () => { + runtimeCall.mockRejectedValue(new Error('terminal_pane_owner_unverified')) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const transport = createRemoteRuntimePtyTransport('hub-env', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + + const result = await transport.connect({ + url: '', + sessionId: 'ssh:hub-private@@pty-2', + callbacks: { onError } + }) + + expect(result).toEqual({ + id: 'ssh:hub-private@@pty-2', + ownerUnverifiable: true + }) + expect(onError).toHaveBeenCalledWith('terminal_pane_owner_unverified') + }) + + it('surfaces a generic saved-pane error without changing its owner verdict', async () => { + runtimeCall.mockRejectedValue(new Error('Remote runtime rejected the pairing token.')) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const transport = createRemoteRuntimePtyTransport('hub-env', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + + const result = await transport.connect({ + url: '', + sessionId: 'ssh:hub-private@@pty-2', + callbacks: { onError } + }) + + expect(result).toBeUndefined() + expect(onError).toHaveBeenCalledWith('Remote runtime rejected the pairing token.') + expect(transport.getRecoveryState?.().phase).toBe('offline') + }) + it('verifies a legacy pane response against the requested worktree session', async () => { runtimeCall.mockImplementation(async (request: { method: string; params?: unknown }) => { if (request.method === 'terminal.resolvePane') { diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index d376897f2fd..977d912cc47 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -81,6 +81,7 @@ import { ptyShutdownLifecycleHandlers } from './pty-shutdown-data-suspension' import { getRuntimeEnvironmentRevision } from '@/runtime/runtime-environment-revision' +import { isTerminalPaneOwnerUnverified } from '../../../../shared/terminal-pane-owner-verdict' const REMOTE_TERMINAL_INPUT_FLUSH_MS = 8 const REMOTE_TERMINAL_VIEWPORT_FLUSH_MS = 33 @@ -2194,21 +2195,21 @@ export function createRemoteRuntimePtyTransport( ...(createdTerminal.isReattach === true ? { isReattach: true } : {}) } satisfies PtyConnectResult } catch (error) { - let ownerUnverifiableResult: PtyConnectResult | undefined + let connectResult: PtyConnectResult | undefined if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) { connecting = false const message = runtimeTerminalErrorMessage(error) if (options.sessionId && isRemoteTerminalGoneMessage(message)) { recovery.cancel() handleRemoteTerminalError(error) - ownerUnverifiableResult = { + connectResult = { id: options.sessionId, exitedBeforeAttach: true } - } else if (options.sessionId) { + } else if (options.sessionId && isTerminalPaneOwnerUnverified(message)) { recovery.markDisconnected() surfaceErrorMessage(message) - ownerUnverifiableResult = { + connectResult = { id: options.sessionId, ownerUnverifiable: true } @@ -2225,7 +2226,7 @@ export function createRemoteRuntimePtyTransport( surfaceErrorMessage(message) } } - return ownerUnverifiableResult + return connectResult } }, diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index e558ad848eb..5a1cc8d1477 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -2934,9 +2934,9 @@ "5c8ce20be6": "If this persists, please", "cc6d997c65": "Restart the terminal daemon from here to clear stale daemon state.", "7ee11bc0db": "Orca couldn't confirm whether this terminal's previous session is still running, so it left the session and history untouched.", - "retryConnection": "Retry connection", - "ownerUnavailableTitle": "Terminal session unavailable", - "ownerUnavailableDetail": "Orca couldn't confirm whether the previous session is still running. The tab and session are preserved; retry when the owner is available.", + "retryConnection": "Check again", + "ownerUnavailableTitle": "Terminal session not connected", + "ownerUnavailableDetail": "Orca couldn't verify whether this saved session is still running, so it left the tab and history untouched. Check again retries the saved panes in this tab; it won't close or replace them.", "e16012e31e": "The terminal daemon that owned this session exited, so the session and its scrollback could not be recovered. Open a new terminal to continue.", "sessionUnavailable": "Orca couldn't reattach to this pane's terminal session on the host. Open a new terminal to continue." }, diff --git a/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts b/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts index ef331ee6277..f3b01a2a536 100644 --- a/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts +++ b/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts @@ -208,6 +208,7 @@ async function runMaterializationJourney( await expect .poll(() => getTerminalContent(page), { timeout: 30_000 }) .toContain('MATERIALIZE_READY') + await expect(page.locator('[data-terminal-owner-unavailable-banner]')).toHaveCount(0) const originalTerminal = await callRuntime<{ terminal: RuntimeTerminalShow }>( page, @@ -315,6 +316,7 @@ async function runMaterializationJourney( await expect(tab).toBeVisible({ timeout: 10_000 }) await tab.click() await expect.poll(() => getTerminalContent(page), { timeout: 10_000 }).toContain(marker) + await expect(page.locator('[data-terminal-owner-unavailable-banner]')).toHaveCount(0) const listed = await callRuntime( page,