fix: harden terminal owner recovery

This commit is contained in:
Jinwoo-H
2026-08-25 05:02:43 -04:00
parent 91ebdfef06
commit a68090a43f
23 changed files with 369 additions and 60 deletions
+40 -8
View File
@@ -14330,22 +14330,24 @@
"platforms": ["macos", "linux", "windows"],
"providers": ["local-daemon", "ssh", "wsl", "remote-runtime"],
"coveredPlatforms": ["macos", "linux"],
"coveredProviders": ["local-daemon", "ssh"],
"coverageNotes": "Deterministic production-boundary tests cover persisted daemon owner identity, full-owner retirement CAS and rollback, local daemon routing, pre-side-effect daemon attach fencing, SSH provider/relay incarnation checks, exact relay exit proof, renderer preservation, and guarded recovery. A real macOS multi-generation daemon journey and rendered Docker SSH reconnect journey passed. Paired headed/headless was blocked by missing web-client URL; targeted A/B update handoff, WSL, ConPTY, and physical Windows remain gaps.",
"coveredProviders": ["local-daemon", "ssh", "remote-runtime"],
"coverageNotes": "Deterministic production-boundary tests cover persisted daemon owner identity, full-owner retirement CAS and rollback, local daemon routing, pre-side-effect daemon attach fencing, SSH provider/relay incarnation checks, exact relay exit proof, renderer preservation, remote-runtime failure classification, and guarded recovery. Real macOS multi-generation, rendered Docker SSH reconnect, and paired remote materialization/reconnect journeys passed. The headed/headless probe-gap journey was blocked by missing web-client URL; targeted A/B update handoff, WSL, ConPTY, and physical Windows remain gaps.",
"motivatingLinks": [
"https://linear.app/stably/issue/STA-5269",
"https://github.com/stablyai/orca/pull/16152"
],
"invariant": "Only the exact execution host, owner incarnation, and session incarnation may report live or exited. Missing routes, adapters, metadata, or contact are unverifiable and preserve every PTY binding, terminal row, pane, tab, and retry lease.",
"oracle": "Persist a PTY owned by daemon A, replace the compatible current slot with daemon B, and have B report the logical id absent. Require an unverifiable verdict without querying B as the owner, without broad fallback, deletion, fresh spawn, or unrelated stop. Exact A plus the exact session may report live or exited; a mismatched reused incarnation, provider loss, and owner-unverified result or throw preserve topology and settle guarded recovery.",
"oracle": "Persist a PTY owned by daemon A, replace the compatible current slot with daemon B, and have B report the logical id absent. Require an unverifiable verdict without querying B as the owner, without broad fallback, deletion, fresh spawn, or unrelated stop. Exact A plus the exact session may report live or exited; an intentional history-preserving stop retains that exact authority and may restore the pane. A mismatched reused incarnation, provider loss, and owner-unverified result or throw preserve topology and settle guarded recovery.",
"commands": [
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-authority.oracle.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-resolution.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/daemon-pty-router-owner-fencing.test.ts src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts src/main/daemon/daemon-pty-adapter-session-adoption.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-incarnation-fence.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts src/main/daemon/daemon-pty-adapter-history-recovery.test.ts src/main/daemon/daemon-pty-history-stop-inventory.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-startup-barrier-and-listing.test.ts src/main/ipc/pty-persisted-incarnation-repair.test.ts src/main/ipc/pty-dead-owner-respawn.test.ts src/main/ipc/pty-controller-spawn-admission.test.ts src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty/pane/stable-owner-retirement.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-spawn.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts src/relay/pty-handler-attach-replay.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts --reporter=dot",
"pnpm exec playwright test tests/e2e/daemon-generation-reconnect-safety.spec.ts tests/e2e/daemon-generation-legacy-close-safety.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1",
"SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1",
"pnpm exec playwright test tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1"
],
@@ -14356,6 +14358,9 @@
"src/main/daemon/daemon-pty-router-owner-fencing.test.ts",
"src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts",
"src/main/daemon/daemon-pty-adapter-session-adoption.test.ts",
"src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts",
"src/main/daemon/daemon-pty-adapter-history-recovery.test.ts",
"src/main/daemon/daemon-pty-history-stop-inventory.test.ts",
"src/main/daemon/terminal-host.test.ts",
"src/main/daemon/terminal-host-incarnation-fence.test.ts",
"src/main/daemon/degraded-daemon-pty-provider.test.ts",
@@ -14374,8 +14379,10 @@
"src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts",
"src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts",
"src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts",
"src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts",
"tests/e2e/daemon-generation-reconnect-safety.spec.ts",
"tests/e2e/daemon-generation-legacy-close-safety.spec.ts",
"tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts",
"tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts",
"tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts",
"tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts"
@@ -14388,6 +14395,24 @@
"only an exact routed owner and session can confirm absence"
]
},
{
"file": "src/main/daemon/daemon-pty-adapter-history-checkpoints.test.ts",
"assertions": [
"a history-preserving stop retains exact owner authority and reports that session exited"
]
},
{
"file": "src/main/daemon/daemon-pty-adapter-history-recovery.test.ts",
"assertions": [
"a delayed physical exit after an unreadable history-preserving stop cannot erase exact exit authority"
]
},
{
"file": "src/main/daemon/daemon-pty-history-stop-inventory.test.ts",
"assertions": [
"authenticated inventory racing a history-preserving exit cannot erase exact exit authority"
]
},
{
"file": "src/main/daemon/terminal-host-incarnation-fence.test.ts",
"assertions": [
@@ -14412,6 +14437,13 @@
"assertions": [
"result and thrown owner-unverified outcomes settle the exact retry and preserve topology"
]
},
{
"file": "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts",
"assertions": [
"runtime outages remain quiet reconnect state while only explicit owner-unverified verdicts show ownership recovery UI",
"generic saved-pane failures retain their original diagnostic without fabricating an owner verdict"
]
}
],
"evidenceRuns": [
@@ -14479,13 +14511,13 @@
"summary": "Three SSH provider and relay files passed 54 attach, incarnation, source-recovery, and exact-exit tests."
},
{
"date": "2026-08-23",
"date": "2026-08-25",
"runner": "local",
"platform": "macos",
"result": "passed",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts --reporter=dot",
"durationSeconds": 4.07,
"summary": "Six renderer files passed 65 liveness, typed-result, topology-preservation, and bounded recovery tests."
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-pane-handle-resolution.test.ts --reporter=dot",
"durationSeconds": 3.72,
"summary": "Seven renderer files passed 85 liveness, failure-classification, topology-preservation, and bounded recovery tests."
},
{
"date": "2026-08-25",
@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { join } from 'node:path'
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { DaemonPtyAdapter } from './daemon-pty-adapter'
import { TerminalSessionExitedError } from './daemon-errors'
import type { DaemonServer } from './daemon-server'
import { getHistorySessionDirName } from './history-paths'
import type { PendingOutputRecord } from './types'
@@ -715,12 +716,13 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
it('writes a final checkpoint before keepHistory shutdown', async () => {
historyAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, historyPath: historyDir })
const { id } = await historyAdapter.spawn({
const spawned = await historyAdapter.spawn({
cols: 80,
rows: 24,
cwd: '/home/user',
sessionId: 'sleep-checkpoint'
})
const { id } = spawned
const checkpointSpy = vi.spyOn(historyAdapter.getHistoryManager()!, 'checkpoint')
lastSubprocess._simulateData('fresh output before sleep\r\n')
@@ -732,6 +734,21 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
{ pendingOutputSeq: expect.any(Number) }
)
expect(existsSync(join(historyDir, getHistorySessionDirName(id)))).toBe(true)
expect(historyAdapter.getTerminalOwnerIdentity(id)).toEqual(spawned.ownerIdentity)
await expect(
historyAdapter.probePtyLiveness(id, spawned.incarnationId, spawned.ownerIdentity)
).resolves.toBe(false)
await expect(
historyAdapter.spawn({
sessionId: id,
attachOnly: true,
expectedIncarnationId: spawned.incarnationId,
expectedIncarnationIsAuthoritative: true,
expectedOwnerIdentity: spawned.ownerIdentity,
cols: 80,
rows: 24
})
).rejects.toBeInstanceOf(TerminalSessionExitedError)
})
itOnPosix('persists final take records that are not represented in the snapshot', async () => {
@@ -146,6 +146,34 @@ describe('DaemonPtyAdapter history recovery', () => {
expect(JSON.parse(readFileSync(metaPath, 'utf-8')).endedAt).toBeNull()
})
it('retains exact exit authority when an unreadable keepHistory stop exits late', async () => {
historyAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, historyPath: historyDir })
const spawned = await historyAdapter.spawn({
cols: 80,
rows: 24,
sessionId: 'sleep-unreadable-late-exit'
})
const reader = (historyAdapter as unknown as { historyReader: HistoryReader }).historyReader
vi.spyOn(reader, 'detectColdRestoreState').mockResolvedValue({
status: 'unreadable',
sessionId: spawned.id
})
const exited = new Promise<void>((resolve) => {
historyAdapter.onExit((event) => {
if (event.id === spawned.id) {
resolve()
}
})
})
await historyAdapter.shutdown(spawned.id, { immediate: true, keepHistory: true })
await exited
await expect(
historyAdapter.probePtyLiveness(spawned.id, spawned.incarnationId, spawned.ownerIdentity)
).resolves.toBe(false)
})
it('suspends an empty final checkpoint with unreadable post-checkpoint recovery', async () => {
historyAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, historyPath: historyDir })
const { id } = await historyAdapter.spawn({
+8 -3
View File
@@ -99,7 +99,10 @@ export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyPro
this.pausedProducerSessionIds.delete(event.sessionId)
this.producerResumesOwedOnReconnect.delete(event.sessionId)
this.backgroundedSessionIds.delete(event.sessionId)
if (!this.sleepRestoreSessionIds.has(event.sessionId)) {
const preservesRestorableAuthority =
this.sleepRestoreSessionIds.has(event.sessionId) ||
this.historyPreservingStopSessionIds.has(event.sessionId)
if (!preservesRestorableAuthority) {
this.coldRestoreCache.delete(event.sessionId)
}
this.sessionsNeedingFullCheckpoint.delete(event.sessionId)
@@ -117,8 +120,10 @@ export class DaemonPtyAdapter extends DaemonPtyDaemonRecovery implements IPtyPro
}
this.initialCwds.delete(event.sessionId)
this.wslDistrosBySessionId.delete(event.sessionId)
this.sessionIncarnations.delete(event.sessionId)
this.sessionOwnerIdentities.delete(event.sessionId)
if (!preservesRestorableAuthority) {
this.sessionIncarnations.delete(event.sessionId)
this.sessionOwnerIdentities.delete(event.sessionId)
}
// oxlint-disable-next-line unicorn/no-useless-spread -- copy-safe: listeners may unsubscribe during iteration
for (const listener of [...this.exitListeners]) {
listener({
@@ -79,6 +79,7 @@ export abstract class DaemonPtyEventSubscriptions extends DaemonPtySessionInvent
this.overlayDeadlineWarnedSessionIds.clear()
this.periodicDeadlineWarnedSessionIds.clear()
this.nonFinalAdmissionDeniedSessionIds.clear()
this.historyPreservingStopSessionIds.clear()
this.coldRestoreCache.clear()
this.wslDistrosBySessionId.clear()
this.pausedProducerSessionIds.clear()
@@ -0,0 +1,47 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { rmSync } from 'node:fs'
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
import type { DaemonServer } from './daemon-server'
import { createMockSubprocess, startDaemonAdapterHarness } from './daemon-pty-adapter-test-harness'
describe('DaemonPtyAdapter history-stop inventory', () => {
let dir: string
let server: DaemonServer
let adapter: DaemonPtyAdapter
let lastSubprocess: ReturnType<typeof createMockSubprocess>
beforeEach(async () => {
const harness = await startDaemonAdapterHarness(() => {
lastSubprocess = createMockSubprocess()
return lastSubprocess
})
dir = harness.dir
server = harness.server
adapter = harness.adapter
})
afterEach(async () => {
adapter.dispose()
await server.shutdown()
rmSync(dir, { recursive: true, force: true })
})
it('retains exact authority when inventory overtakes a history-preserving exit', async () => {
const spawned = await adapter.spawn({ cols: 80, rows: 24 })
const internals = adapter as unknown as {
activeSessionIds: Set<string>
historyPreservingStopSessionIds: Set<string>
}
internals.historyPreservingStopSessionIds.add(spawned.id)
lastSubprocess._simulateExit(0)
internals.activeSessionIds.add(spawned.id)
await adapter.listProcesses()
expect(adapter.hasPty(spawned.id)).toBe(false)
expect(adapter.getTerminalOwnerIdentity(spawned.id)).toEqual(spawned.ownerIdentity)
await expect(
adapter.probePtyLiveness(spawned.id, spawned.incarnationId, spawned.ownerIdentity)
).resolves.toBe(false)
})
})
@@ -2,6 +2,8 @@ import { describe, expect, it, vi } from 'vitest'
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
import { DaemonPtyRouter } from './daemon-pty-router'
import type { PtySpawnOptions, PtySpawnResult } from '../providers/types'
import type { TerminalOwnerIdentity } from '../../shared/terminal-owner-identity'
import { PROTOCOL_VERSION } from './daemon-protocol-version'
type FenceAdapter = DaemonPtyAdapter & {
emitExit: (id: string, code: number, incarnationId: string) => void
@@ -23,6 +25,7 @@ function createFenceAdapter(label: string, initialSessions: string[] = []): Fenc
listProcesses: vi.fn(async () =>
sessions.map((id) => ({ id, incarnationId: `${label}:${id}`, cwd: '', title: label }))
),
probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)),
write: vi.fn(),
shutdown: vi.fn(),
onData: noopSubscription,
@@ -46,6 +49,34 @@ function createFenceAdapter(label: string, initialSessions: string[] = []): Fenc
}
describe('DaemonPtyRouter owner fencing', () => {
it('retains the exact owner route for a history-preserving stop', async () => {
const current = createFenceAdapter('current')
const legacy = createFenceAdapter('legacy', ['parked-session'])
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
await router.discoverLegacySessions()
const parkedOwner = {
executionHostId: 'local',
ownerKind: 'daemon',
ownerIncarnationId: 'legacy-daemon',
sessionIncarnationId: 'legacy:parked-session',
protocolVersion: PROTOCOL_VERSION,
endpointRef: 'daemon-v37'
} satisfies TerminalOwnerIdentity
legacy.getTerminalOwnerIdentity = vi.fn(() => parkedOwner)
legacy.probePtyLiveness = vi.fn(async () => false)
vi.mocked(legacy.shutdown).mockImplementation(async () => {
legacy.emitExit('parked-session', -1, 'legacy:parked-session')
})
await router.shutdown('parked-session', { keepHistory: true })
await expect(
router.probePtyLiveness('parked-session', 'legacy:parked-session', parkedOwner)
).resolves.toBe(false)
expect(legacy.probePtyLiveness).toHaveBeenCalledOnce()
expect(current.probePtyLiveness).not.toHaveBeenCalled()
})
it('keeps replacement B routed after a delayed exit from owner A', async () => {
const current = createFenceAdapter('current')
const legacy = createFenceAdapter('legacy', ['reused-session'])
+7
View File
@@ -29,6 +29,13 @@ export class DaemonPtyRouter implements IPtyProvider {
this.subscriptions = new DaemonPtyAdapterSubscriptionFanout(
this.allAdapters(),
(adapter, payload) => {
const retainedOwner = adapter.getTerminalOwnerIdentity?.(payload.id)
if (
payload.incarnationId &&
retainedOwner?.sessionIncarnationId === payload.incarnationId
) {
return
}
this.ownerResolver.forgetRoute(payload.id, adapter, payload.incarnationId)
},
(adapter) => this.ownerResolver.invalidateProvider(adapter)
@@ -149,6 +149,7 @@ export abstract class DaemonPtyRuntimeState {
protected overlayDeadlineWarnedSessionIds = new Set<string>()
protected periodicDeadlineWarnedSessionIds = new Set<string>()
protected keepHistoryShutdowns = new Set<Promise<void>>()
protected historyPreservingStopSessionIds = new Set<string>()
protected disconnectOnlyPromise: Promise<void> | null = null
protected supportsCheckpoints: boolean
protected supportsIncrementalCheckpoints: boolean
@@ -269,8 +270,12 @@ export abstract class DaemonPtyRuntimeState {
if (!incarnationId || !this.lastAuthenticatedIdentity) {
return
}
const previousIncarnationId = this.sessionIncarnations.get(id)
this.sessionIncarnations.set(id, incarnationId)
this.sessionOwnerIdentities.set(id, { ...this.lastAuthenticatedIdentity })
if (previousIncarnationId && previousIncarnationId !== incarnationId) {
this.historyPreservingStopSessionIds.delete(id)
}
}
protected hasExactSessionAuthority(
+12 -2
View File
@@ -168,6 +168,9 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority
if (opts.keepHistory && this.disconnectOnlyPromise) {
throw new Error('Cannot keep history after daemon disconnect has started')
}
if (opts.keepHistory) {
this.historyPreservingStopSessionIds.add(id)
}
const shutdown = this.withHistorySpawnLock(id, () => this.shutdownWithHistoryLock(id, opts))
if (!opts.keepHistory) {
await shutdown
@@ -176,6 +179,9 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority
this.keepHistoryShutdowns.add(shutdown)
try {
await shutdown
} catch (error) {
this.historyPreservingStopSessionIds.delete(id)
throw error
} finally {
this.keepHistoryShutdowns.delete(shutdown)
}
@@ -242,8 +248,11 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority
remainingDaemonRequestTimeoutMs(opts.deadlineMs)
)
this.activeSessionIds.delete(id)
this.sessionIncarnations.delete(id)
this.sessionOwnerIdentities.delete(id)
if (!opts.keepHistory) {
this.historyPreservingStopSessionIds.delete(id)
this.sessionIncarnations.delete(id)
this.sessionOwnerIdentities.delete(id)
}
this.clearSessionAwaitingDaemonRecovery(id)
this.dirtySessionVersions.delete(id)
if (!opts.keepHistory) {
@@ -282,6 +291,7 @@ export abstract class DaemonPtySessionControl extends DaemonPtySessionAuthority
}
ackColdRestore(sessionId: string): void {
this.historyPreservingStopSessionIds.delete(sessionId)
this.coldRestoreCache.delete(sessionId)
this.sleepRestoreSessionIds.delete(sessionId)
}
@@ -62,8 +62,10 @@ export abstract class DaemonPtySessionInventory extends DaemonPtyProcessInspecti
for (const id of preRequestActiveIds) {
if (!aliveSessionIds.has(id)) {
this.activeSessionIds.delete(id)
this.sessionIncarnations.delete(id)
this.sessionOwnerIdentities.delete(id)
if (!this.historyPreservingStopSessionIds.has(id)) {
this.sessionIncarnations.delete(id)
this.sessionOwnerIdentities.delete(id)
}
}
}
this.publishAuditObservation(
@@ -140,6 +142,7 @@ export abstract class DaemonPtySessionInventory extends DaemonPtyProcessInspecti
this.overlayDeadlineWarnedSessionIds.clear()
this.periodicDeadlineWarnedSessionIds.clear()
this.nonFinalAdmissionDeniedSessionIds.clear()
this.historyPreservingStopSessionIds.clear()
this.pausedProducerSessionIds.clear()
this.producerResumesOwedOnReconnect.clear()
this.stopCheckpointTimer()
@@ -1,5 +1,7 @@
import { recognizeAgentProcessFromCommandLine } from '../../shared/agent-process-recognition'
import { shouldUseShellReadyStartupDelivery } from '../../shared/codex-startup-delivery'
import { sameTerminalOwnerIdentity } from '../../shared/terminal-owner-identity'
import { SessionNotFoundError, TerminalSessionExitedError } from './daemon-errors'
import type {
HistoryRecoveryContext,
PendingDaemonSpawnOperation
@@ -49,6 +51,20 @@ export abstract class DaemonPtySessionSpawn extends DaemonPtySpawnResult {
this.doSpawn({ ...spawnOpts, sessionId }, operation, historyRecovery)
)
)
} catch (error) {
const exactOwner = this.getTerminalOwnerIdentity(sessionId)
if (
error instanceof SessionNotFoundError &&
spawnOpts.attachOnly === true &&
spawnOpts.expectedIncarnationIsAuthoritative === true &&
spawnOpts.expectedIncarnationId !== undefined &&
this.hasExactSessionAuthority(sessionId, spawnOpts.expectedIncarnationId) &&
(spawnOpts.expectedOwnerIdentity === undefined ||
sameTerminalOwnerIdentity(exactOwner, spawnOpts.expectedOwnerIdentity))
) {
throw new TerminalSessionExitedError(sessionId)
}
throw error
} finally {
if (historyRecovery.freeze) {
this.historyManager?.abandonRecoveryFreeze(historyRecovery.freeze)
@@ -75,6 +75,24 @@ describe('SSH PTY provider session reattach incarnation', () => {
)
})
it('rejects a legacy attach that cannot echo a persisted incarnation', async () => {
const mux = {
request: vi.fn().mockResolvedValue({ replay: 'saved output' }),
notify: vi.fn(),
onNotification: vi.fn().mockReturnValue(vi.fn())
}
const provider = new SshPtyProvider('conn-1', mux as never)
await expect(
provider.spawn({
cols: 80,
rows: 24,
sessionId: 'pty-old',
expectedIncarnationId: 'incarnation-original'
})
).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError)
})
it('keeps relay absence owner-unverified', async () => {
const mux = {
request: vi.fn().mockRejectedValue(new Error('PTY "pty-old" not found')),
@@ -191,7 +191,7 @@ export function TerminalErrorToast({
<div className="shrink-0 text-sm font-semibold">
{translate(
'auto.components.terminal.pane.TerminalErrorToast.ownerUnavailableTitle',
'Terminal session unavailable'
'Terminal session not connected'
)}
</div>
<div className="flex min-w-0 items-center gap-1.5 text-xs text-muted-foreground">
@@ -201,7 +201,7 @@ export function TerminalErrorToast({
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
{translate(
'auto.components.terminal.pane.TerminalErrorToast.ownerUnavailableDetail',
"Orca couldn't confirm whether the previous session is still running. The tab and session are preserved; retry when the owner is available."
"Orca couldn't verify whether this saved session is still running, so it left the tab and history untouched. Check again retries the saved panes in this tab; it won't close or replace them."
)}
</div>
{footer ? (
@@ -213,7 +213,7 @@ export function TerminalErrorToast({
<Button className="shrink-0" size="sm" onClick={onRetry}>
{translate(
'auto.components.terminal.pane.TerminalErrorToast.retryConnection',
'Retry connection'
'Check again'
)}
</Button>
</div>
@@ -64,7 +64,6 @@ import {
stripSshReconnectOwnedErrorLines,
TerminalErrorToast
} from './TerminalErrorToast'
import { TERMINAL_PANE_OWNER_UNVERIFIED } from '../../../../shared/terminal-pane-owner-verdict'
import { TerminalProcessExitOverlay } from './TerminalProcessExitOverlay'
import { TerminalSessionStateSaveFailureDialog } from './TerminalSessionStateSaveFailureDialog'
import TerminalContextMenu from './TerminalContextMenu'
@@ -507,8 +506,9 @@ function TerminalPane(
}
}, [])
const retryTerminalConnection = useCallback(() => {
setTerminalError(null)
useAppStore.getState().remountTerminalTabForRecovery(tabId)
if (useAppStore.getState().remountTerminalTabForRecovery(tabId)) {
setTerminalError(null)
}
}, [tabId])
const onPtyRecoveryStateRef = useRef(
(paneId: number, state: PtyTransportRecoveryState | null) => {
@@ -2932,32 +2932,6 @@ function TerminalPane(
const activePane = managerRef.current?.getActivePane()
const managedPanes = managerRef.current?.getPanes() ?? []
// A daemon handoff can leave the renderer waiting on a preserved PTY without
// producing an IPC error. Surface the same explicit retry state instead of
// leaving the user with an unexplained blank pane.
useEffect(() => {
const currentPanes = managerRef.current?.getPanes() ?? []
if (!isActive || !isVisible || currentPanes.length === 0) {
return
}
const savedPtyIds = Object.values(savedLayout.ptyIdsByLeafId ?? {}).filter(Boolean)
if (savedPtyIds.length === 0) {
return
}
const hasUnboundPane = currentPanes.some(
(pane) => paneTransportsRef.current.get(pane.id)?.getPtyId() == null
)
if (!hasUnboundPane) {
if (terminalError && isPaneOwnerUnverifiedError(terminalError)) {
setTerminalError(null)
}
return
}
const timer = window.setTimeout(() => {
setTerminalError((current) => current ?? TERMINAL_PANE_OWNER_UNVERIFIED)
}, 1_500)
return () => window.clearTimeout(timer)
}, [isActive, isVisible, managerRef, paneCount, paneLayoutRevision, savedLayout, terminalError])
const showSshReconnectOverlay = Boolean(
isActive &&
isVisible &&
@@ -403,6 +403,7 @@ describe('connectPanePty', () => {
expect(deps.clearTabPtyId).not.toHaveBeenCalled()
expect(deps.syncPanePtyLayoutBinding).not.toHaveBeenCalled()
expect(deps.updateTabPtyId).not.toHaveBeenCalled()
expect(deps.onPtyErrorRef.current).not.toHaveBeenCalled()
})
it.each([
@@ -474,8 +475,39 @@ describe('connectPanePty', () => {
{ id: 'tab-1', ptyId: null, generation: 7 }
])
expect(mockStoreState.deferredSshSessionIdsByTabId['tab-1']).toBe(restoredPtyId)
if (outcome === 'reject') {
expect(deps.onPtyErrorRef.current).toHaveBeenCalledWith(1, 'relay attach timed out')
} else {
expect(deps.onPtyErrorRef.current).not.toHaveBeenCalled()
}
expect(deps.onPtyErrorRef.current).not.toHaveBeenCalledWith(
expect.anything(),
expect.stringContaining('terminal_pane_owner_unverified')
)
}
)
it('surfaces a generic restored-pane failure without calling it owner-unverified', async () => {
const { connectPanePty } = await import('./pty-connection')
const transport = createMockTransport()
transport.connect.mockRejectedValueOnce(new Error('daemon package is unavailable'))
transportFactoryQueue.push(transport)
const deps = createDeps({
restoredLeafId: LEAF_2,
restoredPtyIdByLeafId: { [LEAF_2]: 'preserved-pty' }
})
connectPanePty(createPane(2) as never, createManager(2) as never, deps as never)
await flushAsyncTicks()
expect(deps.onPtyErrorRef.current).toHaveBeenCalledWith(2, 'daemon package is unavailable')
expect(deps.onPtyErrorRef.current).not.toHaveBeenCalledWith(
expect.anything(),
expect.stringContaining('terminal_pane_owner_unverified')
)
expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled()
expect(deps.clearTabPtyId).not.toHaveBeenCalled()
})
it.each(['result', 'throw'] as const)(
'preserves a restored pane binding when its daemon owner is unverifiable (%s)',
async (outcome) => {
@@ -15,6 +15,7 @@ import type { ConnectPanePtySession } from './connect-pane-pty-session'
import { runDeferredSessionReattachChoice } from './deferred-session-reattach-choice'
import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery'
import { isTerminalPaneOwnerUnverified } from '../../../../../shared/terminal-pane-owner-verdict'
export function runDeferredSessionAttach(session: ConnectPanePtySession): void {
// Why: trigger the deferred SSH connect per-tab (not per-target) so multiple tabs for one target reattach independently.
@@ -291,8 +292,13 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void {
})
return
}
session.reportError(err instanceof Error ? err.message : String(err))
recoverUnverifiableReattach(session, pendingSessionId)
const message = err instanceof Error ? err.message : String(err)
session.reportError(message)
if (isTerminalPaneOwnerUnverified(message)) {
recoverUnverifiableReattach(session, pendingSessionId)
} else if (session.directSshRetryAttempt) {
session.settleDirectSshPaneRetryAttempt(session.directSshRetryAttempt, 'failed')
}
})
session.armDirectSshPaneRetryTimeout(
trackedReattachPromise,
@@ -165,7 +165,9 @@ export function startDeferredSessionReattach(
return
}
session.reportError(message)
recoverUnverifiableReattach(session, deferredReattachSessionId)
if (session.directSshRetryAttempt) {
session.settleDirectSshPaneRetryAttempt(session.directSshRetryAttempt, 'failed')
}
})
session.armDirectSshPaneRetryTimeout(trackedReattachPromise, session.directSshRetryAttempt)
}
@@ -42,6 +42,7 @@ type ReattachResultSession = ReattachPayloadSession &
| 'scheduleReattachIdleAgentCursorReset'
| 'serializeHiddenOutputSnapshot'
| 'setPanePtyFitBinding'
| 'settleDirectSshPaneRetryAttempt'
| 'startFreshColdRestoreAgentResume'
| 'structuralReplayCoordinator'
| 'syncHiddenRendererPtyDelivery'
@@ -95,7 +96,9 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi
paneId: session.pane.id,
ptyId: staleSessionId ?? null
})
recoverUnverifiableReattach(sessionBag, staleSessionId)
if (session.directSshRetryAttempt) {
session.settleDirectSshPaneRetryAttempt(session.directSshRetryAttempt, 'failed')
}
return false
}
session.registerEffectiveLaunchConfig(connectResult?.launchConfig, {
@@ -96,6 +96,75 @@ describe('createRemoteRuntimePtyTransport', () => {
expect(latestSubscribePayload()).toMatchObject({ terminal: 'hub-terminal-1' })
})
it('keeps a saved host pane in reconnect recovery for a runtime outage', async () => {
runtimeCall.mockRejectedValue(
Object.assign(new Error('Remote Orca runtime closed the connection.'), {
code: 'remote_runtime_unavailable'
})
)
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('hub-env', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
const result = await transport.connect({
url: '',
sessionId: 'ssh:hub-private@@pty-2',
callbacks: { onError }
})
expect(result).toBeUndefined()
expect(onError).not.toHaveBeenCalled()
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
})
it('returns owner-unverifiable only for an explicit saved-pane owner verdict', async () => {
runtimeCall.mockRejectedValue(new Error('terminal_pane_owner_unverified'))
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('hub-env', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
const result = await transport.connect({
url: '',
sessionId: 'ssh:hub-private@@pty-2',
callbacks: { onError }
})
expect(result).toEqual({
id: 'ssh:hub-private@@pty-2',
ownerUnverifiable: true
})
expect(onError).toHaveBeenCalledWith('terminal_pane_owner_unverified')
})
it('surfaces a generic saved-pane error without changing its owner verdict', async () => {
runtimeCall.mockRejectedValue(new Error('Remote runtime rejected the pairing token.'))
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
const onError = vi.fn()
const transport = createRemoteRuntimePtyTransport('hub-env', {
worktreeId: 'wt-1',
tabId: 'tab-1',
leafId: 'pane:1'
})
const result = await transport.connect({
url: '',
sessionId: 'ssh:hub-private@@pty-2',
callbacks: { onError }
})
expect(result).toBeUndefined()
expect(onError).toHaveBeenCalledWith('Remote runtime rejected the pairing token.')
expect(transport.getRecoveryState?.().phase).toBe('offline')
})
it('verifies a legacy pane response against the requested worktree session', async () => {
runtimeCall.mockImplementation(async (request: { method: string; params?: unknown }) => {
if (request.method === 'terminal.resolvePane') {
@@ -81,6 +81,7 @@ import {
ptyShutdownLifecycleHandlers
} from './pty-shutdown-data-suspension'
import { getRuntimeEnvironmentRevision } from '@/runtime/runtime-environment-revision'
import { isTerminalPaneOwnerUnverified } from '../../../../shared/terminal-pane-owner-verdict'
const REMOTE_TERMINAL_INPUT_FLUSH_MS = 8
const REMOTE_TERMINAL_VIEWPORT_FLUSH_MS = 33
@@ -2194,21 +2195,21 @@ export function createRemoteRuntimePtyTransport(
...(createdTerminal.isReattach === true ? { isReattach: true } : {})
} satisfies PtyConnectResult
} catch (error) {
let ownerUnverifiableResult: PtyConnectResult | undefined
let connectResult: PtyConnectResult | undefined
if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) {
connecting = false
const message = runtimeTerminalErrorMessage(error)
if (options.sessionId && isRemoteTerminalGoneMessage(message)) {
recovery.cancel()
handleRemoteTerminalError(error)
ownerUnverifiableResult = {
connectResult = {
id: options.sessionId,
exitedBeforeAttach: true
}
} else if (options.sessionId) {
} else if (options.sessionId && isTerminalPaneOwnerUnverified(message)) {
recovery.markDisconnected()
surfaceErrorMessage(message)
ownerUnverifiableResult = {
connectResult = {
id: options.sessionId,
ownerUnverifiable: true
}
@@ -2225,7 +2226,7 @@ export function createRemoteRuntimePtyTransport(
surfaceErrorMessage(message)
}
}
return ownerUnverifiableResult
return connectResult
}
},
+3 -3
View File
@@ -2934,9 +2934,9 @@
"5c8ce20be6": "If this persists, please",
"cc6d997c65": "Restart the terminal daemon from here to clear stale daemon state.",
"7ee11bc0db": "Orca couldn't confirm whether this terminal's previous session is still running, so it left the session and history untouched.",
"retryConnection": "Retry connection",
"ownerUnavailableTitle": "Terminal session unavailable",
"ownerUnavailableDetail": "Orca couldn't confirm whether the previous session is still running. The tab and session are preserved; retry when the owner is available.",
"retryConnection": "Check again",
"ownerUnavailableTitle": "Terminal session not connected",
"ownerUnavailableDetail": "Orca couldn't verify whether this saved session is still running, so it left the tab and history untouched. Check again retries the saved panes in this tab; it won't close or replace them.",
"e16012e31e": "The terminal daemon that owned this session exited, so the session and its scrollback could not be recovered. Open a new terminal to continue.",
"sessionUnavailable": "Orca couldn't reattach to this pane's terminal session on the host. Open a new terminal to continue."
},
@@ -208,6 +208,7 @@ async function runMaterializationJourney(
await expect
.poll(() => getTerminalContent(page), { timeout: 30_000 })
.toContain('MATERIALIZE_READY')
await expect(page.locator('[data-terminal-owner-unavailable-banner]')).toHaveCount(0)
const originalTerminal = await callRuntime<{ terminal: RuntimeTerminalShow }>(
page,
@@ -315,6 +316,7 @@ async function runMaterializationJourney(
await expect(tab).toBeVisible({ timeout: 10_000 })
await tab.click()
await expect.poll(() => getTerminalContent(page), { timeout: 10_000 }).toContain(marker)
await expect(page.locator('[data-terminal-owner-unavailable-banner]')).toHaveCount(0)
const listed = await callRuntime<RuntimeTerminalListResult>(
page,