From c86418eaad20126430dcb3128a4a61d438feda61 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:53:17 -0700 Subject: [PATCH 001/167] rm git shim (#14141) * rm git shim Drops the terminal git/gh PATH wrapper and its settings toggle. Renames the no-marker shell-ready launch config after what it does. Co-authored-by: Orca * rm git shim: clear stale state from older installs Deletes the orphaned wrapper dir and scrubs inherited env/PATH, so a daemon that outlives the upgrade cannot keep seeding it. Drops a now-unread spawn option. Co-authored-by: Orca * rm git shim: cover the daemon and headless paths Scrub after the PATH prepends (they re-read process.env on the sparse daemon env) and run the cleanup above the serve branch so remote hosts get it too. Retry a locked removal; match PATH case-insensitively. Co-authored-by: Orca * rm git shim: keep the scrub final Refuse to re-prepend a legacy entry during agent-teams PATH promotion, which runs after the scrub. Cover the removal guard. Co-authored-by: Orca --------- Co-authored-by: Orca --- config/max-lines-baseline.txt | 1 - config/scripts/locale-ko-key-overrides.json | 9 - config/scripts/locale-ko-value-overrides.mjs | 4 +- .../locale-search-keyword-overrides.mjs | 1 - .../locale-translation-policy.es-pr.test.mjs | 6 +- ...cale-translation-policy.zh-round5.test.mjs | 14 +- .../scripts/locale-zh-phrase-fixes-round5.mjs | 1 - config/scripts/locale-zh-value-overrides.mjs | 4 +- ...rminal-attribution-path-env-casing.test.ts | 169 --- ...al-attribution-win32-dash-argument.test.ts | 100 -- .../attribution/terminal-attribution.test.ts | 824 ------------- src/main/attribution/terminal-attribution.ts | 1058 ----------------- .../runtime-home-service.test.ts | 1 - src/main/codex-accounts/service.test.ts | 1 - src/main/daemon/pty-subprocess.test.ts | 8 +- src/main/daemon/pty-subprocess.ts | 7 +- src/main/daemon/shell-ready.test.ts | 6 +- src/main/daemon/shell-ready.ts | 28 +- src/main/index.ts | 3 + src/main/ipc/pty.test.ts | 203 ++-- src/main/ipc/pty.ts | 43 +- src/main/providers/local-pty-provider.test.ts | 12 +- src/main/providers/local-pty-provider.ts | 11 +- .../providers/local-pty-shell-ready.test.ts | 10 +- src/main/providers/local-pty-shell-ready.ts | 28 +- src/main/providers/ssh-pty-provider.test.ts | 6 +- src/main/pty/legacy-terminal-shim-dir.test.ts | 124 ++ src/main/pty/legacy-terminal-shim-dir.ts | 71 ++ .../claude-agent-teams-shim-env.test.ts | 2 +- .../runtime/claude-agent-teams-shim-env.ts | 2 +- .../claude-agent-teams-tmux-dispatcher.ts | 4 +- src/main/runtime/orca-runtime-git.test.ts | 6 +- src/main/runtime/orca-runtime-git.ts | 5 +- src/main/runtime/rpc/methods/git-params.ts | 1 - src/main/runtime/rpc/methods/git.test.ts | 2 - src/main/runtime/rpc/methods/git.ts | 6 - .../commit-message-text-generation.test.ts | 1 - src/relay/pty-handler.test.ts | 40 +- .../src/components/settings/GitPane.tsx | 49 - .../src/components/settings/Settings.tsx | 2 +- .../src/components/settings/git-search.ts | 19 +- .../hooks/useSettingsNavigationMetadata.ts | 2 +- src/renderer/src/i18n/locales/en.json | 20 +- src/renderer/src/i18n/locales/es.json | 20 +- src/renderer/src/i18n/locales/ja.json | 20 +- src/renderer/src/i18n/locales/ko.json | 20 +- src/renderer/src/i18n/locales/zh.json | 20 +- .../src/runtime/runtime-git-client.test.ts | 4 +- .../src/runtime/runtime-git-client.ts | 17 +- src/shared/commit-message-prompt.ts | 2 +- src/shared/constants.ts | 1 - src/shared/external-worktree-inbox.test.ts | 1 - src/shared/orca-attribution.ts | 6 - src/shared/types.ts | 1 - src/shared/worktree-ownership.test.ts | 1 - 55 files changed, 366 insertions(+), 2661 deletions(-) delete mode 100644 src/main/attribution/terminal-attribution-path-env-casing.test.ts delete mode 100644 src/main/attribution/terminal-attribution-win32-dash-argument.test.ts delete mode 100644 src/main/attribution/terminal-attribution.test.ts delete mode 100644 src/main/attribution/terminal-attribution.ts create mode 100644 src/main/pty/legacy-terminal-shim-dir.test.ts create mode 100644 src/main/pty/legacy-terminal-shim-dir.ts delete mode 100644 src/shared/orca-attribution.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 59368273624..a9c03d1b9d6 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -10,7 +10,6 @@ inline src/main/agent-hooks/server.test.ts inline src/main/agent-hooks/server.ts inline src/main/amp/hook-service.ts inline src/main/antigravity/hook-service.ts -inline src/main/attribution/terminal-attribution.ts inline src/main/automations/external-manager.ts inline src/main/automations/hermes-cron-output.ts inline src/main/browser/agent-browser-bridge.test.ts diff --git a/config/scripts/locale-ko-key-overrides.json b/config/scripts/locale-ko-key-overrides.json index bd0c3c15aee..aacf053926e 100644 --- a/config/scripts/locale-ko-key-overrides.json +++ b/config/scripts/locale-ko-key-overrides.json @@ -2273,12 +2273,6 @@ "auto.components.settings.GitPane.976afc6b3e": { "ko": "워크스페이스를 만들면 Orca가 원격 기준 브랜치를 새로고침하고, 일치하는 로컬 브랜치를 안전하게 fast-forward합니다. 예:" }, - "auto.components.settings.GitPane.b5f534717a": { - "ko": "공동 작성" - }, - "auto.components.settings.GitPane.b9b5771bb1": { - "ko": "표기" - }, "auto.components.settings.GitPane.b9c011fbc2": { "ko": "속도 제한" }, @@ -2288,9 +2282,6 @@ "auto.components.settings.GitPane.db3a127eb1": { "ko": "이렇게 하면 다음과 같은 명령이 유지됩니다" }, - "auto.components.settings.GitPane.e02ea23a32": { - "ko": "Orca 표기" - }, "auto.components.settings.GitPane.f35007e6e8": { "ko": "git-username" }, diff --git a/config/scripts/locale-ko-value-overrides.mjs b/config/scripts/locale-ko-value-overrides.mjs index 19db7c753ec..5dac8b29a0d 100644 --- a/config/scripts/locale-ko-value-overrides.mjs +++ b/config/scripts/locale-ko-value-overrides.mjs @@ -44,7 +44,6 @@ export const KO_VALUE_OVERRIDES = { 'Issue #{{value0}}': '이슈 #{{value0}}', 'Linked issue #{{value0}}': '연결된 이슈 #{{value0}}', 'Issue # or GitHub URL': '이슈 # 또는 GitHub URL', - 'Add Orca attribution to commits, PRs, and issues.': '커밋, PR 및 이슈에 Orca 속성을 추가합니다.', 'Paste an issue URL, or enter a number. Leave blank to remove the link.': '이슈 URL을 붙여넣거나 숫자를 입력하세요. 링크를 제거하려면 비워두세요.', 'Connect Linear in Settings to search issues.': '이슈를 검색하려면 설정에서 Linear를 연결하세요.', @@ -157,8 +156,7 @@ export const KO_VALUE_OVERRIDES = { '저장되지 않은 Git AI Author 변경사항이 있습니다. 떠나면 폐기됩니다.', 'Discard unsaved Git AI Author changes?': '저장되지 않은 Git AI Author 변경사항을 삭제하시겠습니까?', - 'Branch naming, base refs, attribution, and Git AI Author.': - '브랜치 이름 지정, 기본 참조, 속성 및 Git AI Author.', + 'Branch naming, base refs, and Git AI Author.': '브랜치 이름 지정, 기본 참조 및 Git AI Author.', 'Used by repositories that inherit global hosted-review defaults.': '전역 호스팅 PR 기본값을 상속하는 저장소에서 사용됩니다.', 'Defaults used when the hosted-review composer opens.': diff --git a/config/scripts/locale-search-keyword-overrides.mjs b/config/scripts/locale-search-keyword-overrides.mjs index 2a9d6266f5d..352199c1f7e 100644 --- a/config/scripts/locale-search-keyword-overrides.mjs +++ b/config/scripts/locale-search-keyword-overrides.mjs @@ -164,7 +164,6 @@ export const SEARCH_KEYWORD_OVERRIDES = { gitlab: 'GitLab', gitea: 'Gitea', oauth: 'OAuth', - attribution: '归因', skill: '技能', skills: '技能', phone: '手机', diff --git a/config/scripts/locale-translation-policy.es-pr.test.mjs b/config/scripts/locale-translation-policy.es-pr.test.mjs index a3c046ce895..389f914c86b 100644 --- a/config/scripts/locale-translation-policy.es-pr.test.mjs +++ b/config/scripts/locale-translation-policy.es-pr.test.mjs @@ -25,10 +25,10 @@ describe('locale-translation-policy es PR glossary', () => { it('rewrites PR inside longer sentences', () => { expect( repairEs( - 'Add Orca attribution to commits, PRs, and issues.', - 'Agregue la atribución de Orca a commits, relaciones públicas y problemas.' + 'Split a large change into smaller PRs', + 'Dividir un cambio grande en relaciones públicas más pequeños' ) - ).toBe('Agregue la atribución de Orca a commits, PR y problemas.') + ).toBe('Dividir un cambio grande en PR más pequeños') expect( repairEs( 'Open the PR details to view current reviewers.', diff --git a/config/scripts/locale-translation-policy.zh-round5.test.mjs b/config/scripts/locale-translation-policy.zh-round5.test.mjs index 28901fba8d4..622b3c5c31c 100644 --- a/config/scripts/locale-translation-policy.zh-round5.test.mjs +++ b/config/scripts/locale-translation-policy.zh-round5.test.mjs @@ -44,22 +44,14 @@ describe('locale-translation-policy zh round 5', () => { locale: 'zh' }) ).toBe('显示 Orca Mobile 按钮') - expect( - repairTranslatedValue({ - key: 'auto.components.settings.GitPane.e02ea23a32', - enValue: 'Orca Attribution', - localeValue: 'Orca归属', - locale: 'zh' - }) - ).toBe('Orca 归因') expect( repairTranslatedValue({ key: 'auto.hooks.useSettingsNavigationMetadata.ab4b21b58e', - enValue: 'Branch naming, base refs, attribution, and Git AI Author.', - localeValue: '分支命名、基本引用、归属和 Git AI 作者。', + enValue: 'Branch naming, base refs, and Git AI Author.', + localeValue: '分支命名、基本引用和 Git AI 作者。', locale: 'zh' }) - ).toBe('分支命名、基础引用、归因和 Git AI Author。') + ).toBe('分支命名、基础引用和 Git AI Author。') expect( repairTranslatedValue({ key: 'auto.components.workspace.cleanup.WorkspaceCleanupDialog.1b18868569', diff --git a/config/scripts/locale-zh-phrase-fixes-round5.mjs b/config/scripts/locale-zh-phrase-fixes-round5.mjs index 1aa117395e8..23fcc6d32a5 100644 --- a/config/scripts/locale-zh-phrase-fixes-round5.mjs +++ b/config/scripts/locale-zh-phrase-fixes-round5.mjs @@ -3,7 +3,6 @@ export const ZH_PHRASE_FIXES_ROUND5 = [ { pattern: /Orca集成开发环境/g, replacement: 'Orca IDE', whenEnIncludes: 'Orca IDE' }, { pattern: /Orca第一/g, replacement: 'Orca 优先', whenEnIncludes: 'Orca first' }, { pattern: /Orca移动/g, replacement: 'Orca Mobile', whenEnIncludes: 'Orca Mobile' }, - { pattern: /Orca归属/g, replacement: 'Orca 归因', whenEnIncludes: 'Orca Attribution' }, { pattern: /Orca标志/g, replacement: 'Orca 标志', whenEnIncludes: 'Orca logo' }, { pattern: /喜欢Orca/g, replacement: '喜欢 Orca', whenEnIncludes: 'Enjoying Orca' }, { pattern: /认识Orca/g, replacement: '了解 Orca', whenEnIncludes: 'Get to know Orca' }, diff --git a/config/scripts/locale-zh-value-overrides.mjs b/config/scripts/locale-zh-value-overrides.mjs index 087091a3f3e..b53fb1d2248 100644 --- a/config/scripts/locale-zh-value-overrides.mjs +++ b/config/scripts/locale-zh-value-overrides.mjs @@ -144,7 +144,6 @@ export const ZH_VALUE_OVERRIDES = { 'Expand Orca': '展开 Orca', 'Support Orca': '支持 Orca', 'Restarting Orca…': '正在重启 Orca…', - 'Orca Attribution': 'Orca 归因', 'Show Orca Mobile Button': '显示 Orca Mobile 按钮', 'Codex Accounts': 'Codex 账户', 'Codex Account': 'Codex 账户', @@ -186,8 +185,7 @@ export const ZH_VALUE_OVERRIDES = { "Give agents direct access to Orca's browser so they can test pages, capture screenshots, and act on what they see.": '让代理直接访问 Orca 的浏览器，以便测试页面、捕获屏幕截图并根据所见内容执行操作。', 'X finishes, send it the review task.”': 'X 完成后，把评审任务发给它。”', - 'Branch naming, base refs, attribution, and Git AI Author.': - '分支命名、基础引用、归因和 Git AI Author。', + 'Branch naming, base refs, and Git AI Author.': '分支命名、基础引用和 Git AI Author。', 'You have unsaved Git AI Author changes. Leaving will discard them.': '您有未保存的 Git AI Author 更改。离开将丢弃它们。', 'Discard unsaved Git AI Author changes?': '放弃未保存的 Git AI Author 更改？', diff --git a/src/main/attribution/terminal-attribution-path-env-casing.test.ts b/src/main/attribution/terminal-attribution-path-env-casing.test.ts deleted file mode 100644 index 95ff6fdc2e3..00000000000 --- a/src/main/attribution/terminal-attribution-path-env-casing.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { applyTerminalAttributionEnv } from './terminal-attribution' - -describe('applyTerminalAttributionEnv PATH key casing', () => { - let tmpRoot: string | null = null - - afterEach(() => { - if (tmpRoot) { - rmSync(tmpRoot, { force: true, recursive: true }) - tmpRoot = null - } - }) - - function makeTmpRoot(): string { - tmpRoot = mkdtempSync(join(tmpdir(), 'orca-attribution-path-casing-')) - return tmpRoot - } - - // Why: a Windows child that inherits both spellings crashes the packaged orca.exe launcher - // with "Item has already been added. Key in dictionary: 'PATH'" (stablyai/orca#12046). - function pathEnvKeys(env: Record): string[] { - return Object.keys(env).filter((key) => /^path$/i.test(key)) - } - - it('writes back the inherited Windows `Path` spelling instead of adding `PATH`', () => { - const root = makeTmpRoot() - const userDataPath = join(root, 'user-data') - const baseEnv: Record = { - Path: 'C:\\Windows\\system32;C:\\Program Files\\Git\\cmd', - SystemRoot: 'C:\\Windows' - } - - applyTerminalAttributionEnv(baseEnv, { - enabled: true, - platform: 'win32', - shellFamily: 'native-windows', - userDataPath - }) - - const win32Dir = join(userDataPath, 'orca-terminal-attribution', 'win32') - const pathEntries = baseEnv.Path.split(';') - - expect(pathEnvKeys(baseEnv)).toEqual(['Path']) - expect(pathEntries[0]).toBe(win32Dir) - expect(pathEntries).toContain('C:\\Program Files\\Git\\cmd') - }) - - // Why: Win32 resolves a duplicated variable by taking the first match in the block, so the - // shim must land on the first-listed spelling and the shadowed one must keep its value — - // dropping it would promote it the moment anything downstream removes the live key. - it.each([ - { first: 'PATH', second: 'Path' }, - { first: 'Path', second: 'PATH' } - ])( - 'prepends the shim onto the first-listed `$first` and leaves `$second` untouched', - ({ first, second }) => { - const root = makeTmpRoot() - const userDataPath = join(root, 'user-data') - const baseEnv: Record = { - [first]: 'C:\\tools\\bin;C:\\Windows\\system32', - [second]: 'C:\\Windows\\system32' - } - - applyTerminalAttributionEnv(baseEnv, { - enabled: true, - platform: 'win32', - shellFamily: 'native-windows', - userDataPath - }) - - const win32Dir = join(userDataPath, 'orca-terminal-attribution', 'win32') - expect(baseEnv[first].split(';')).toEqual([ - win32Dir, - 'C:\\tools\\bin', - 'C:\\Windows\\system32' - ]) - expect(baseEnv[second]).toBe('C:\\Windows\\system32') - } - ) - - it('keeps a shadowed duplicate when the live spelling strips down to nothing', () => { - const root = makeTmpRoot() - const userDataPath = join(root, 'user-data') - const shimDir = join(userDataPath, 'orca-terminal-attribution', 'win32') - const baseEnv: Record = { Path: shimDir, PATH: 'C:\\Windows\\system32' } - - applyTerminalAttributionEnv(baseEnv, { - enabled: false, - platform: 'win32', - shellFamily: 'native-windows', - userDataPath - }) - - expect(pathEnvKeys(baseEnv)).toEqual(['PATH']) - expect(baseEnv.PATH).toBe('C:\\Windows\\system32') - }) - - it('strips shim entries from the inherited `Path` when attribution is disabled', () => { - const root = makeTmpRoot() - const userDataPath = join(root, 'user-data') - const shimDir = join(userDataPath, 'orca-terminal-attribution', 'win32') - const baseEnv: Record = { Path: `${shimDir};C:\\Windows\\system32` } - - applyTerminalAttributionEnv(baseEnv, { - enabled: false, - platform: 'win32', - shellFamily: 'native-windows', - userDataPath - }) - - expect(pathEnvKeys(baseEnv)).toEqual(['Path']) - expect(baseEnv.Path).toBe('C:\\Windows\\system32') - }) - - // Why: the daemon receives a sparse env patch with no path key and re-merges its own block - // underneath it, so guessing the spelling here hands the child both. - it.each([ - { hostKey: 'PATH', otherKey: 'Path' }, - { hostKey: 'Path', otherKey: 'PATH' } - ])('adopts a host block spelt `$hostKey` on a path-less env', ({ hostKey, otherKey }) => { - const root = makeTmpRoot() - const userDataPath = join(root, 'user-data') - const inherited = { PATH: process.env.PATH, Path: process.env.Path } - delete process.env.PATH - delete process.env.Path - process.env[hostKey] = 'C:\\Windows\\system32' - - try { - const baseEnv: Record = { SystemRoot: 'C:\\Windows' } - - applyTerminalAttributionEnv(baseEnv, { - enabled: true, - platform: 'win32', - shellFamily: 'native-windows', - userDataPath - }) - - expect(pathEnvKeys(baseEnv)).toEqual([hostKey]) - expect(baseEnv[otherKey]).toBeUndefined() - } finally { - delete process.env[hostKey] - for (const [key, value] of Object.entries(inherited)) { - if (value !== undefined) { - process.env[key] = value - } - } - } - }) - - it('leaves a case-sensitive POSIX `Path` variable alone', () => { - const root = makeTmpRoot() - const baseEnv: Record = { - PATH: '/usr/bin:/bin', - Path: '/decoy' - } - - applyTerminalAttributionEnv(baseEnv, { - enabled: true, - platform: 'linux', - userDataPath: join(root, 'user-data') - }) - - expect(baseEnv.Path).toBe('/decoy') - expect(baseEnv.PATH.split(':')).toContain('/usr/bin') - }) -}) diff --git a/src/main/attribution/terminal-attribution-win32-dash-argument.test.ts b/src/main/attribution/terminal-attribution-win32-dash-argument.test.ts deleted file mode 100644 index 810322a53c9..00000000000 --- a/src/main/attribution/terminal-attribution-win32-dash-argument.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { applyTerminalAttributionEnv } from './terminal-attribution' - -// Why (#12046 follow-up): `powershell.exe -File + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md new file mode 100644 index 00000000000..30d99ae116e --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md @@ -0,0 +1,186 @@ +# Design brief — terminal session ownership & transport + +Review artifact. Repo: Orca (Electron + renderer + relay). Everything below marked +**[V]** was verified by reading the code at the cited location. Everything marked +**[R]** is agent-reported and only spot-checked. Treat **[R]** as a lead, not a fact. + +--- + +## 1. The product problem + +Orca runs terminals on four "routes": + +| Route | Provider | Survives app restart | +| --------------------- | ----------------------------------------------- | -------------------------- | +| local in-process | `LocalPtyProvider` | no | +| local daemon | `DaemonPtyAdapter` (swapped into the same slot) | yes | +| direct SSH | `SshPtyProvider` → remote relay | yes, within a grace window | +| paired remote runtime | not an `IPtyProvider`; runtime RPC | yes | + +Reported failures: + +1. **Pane cardinality growth (STA-3077).** SSH reconnect: 2 terminals → 19 → 20. + Most are panes the user never opened. Remote host fills with unused shells. +2. **Duplicate agent resume.** A coding agent gets resumed twice, two processes + writing one conversation file. One report reached five. +3. **Blast radius.** One PTY failing to re-prove its output stream could drop the + whole host connection, killing every pane, file transfer and git command on it. + +## 2. Verified facts (the evidence base) + +**[V] Dispatch is a substitution for daemon, a parallel registry for SSH.** +`src/main/ipc/pty.ts:932-941`: + +```ts +function getProvider(connectionId) { + if (!connectionId) return localProvider // local AND daemon — one slot + const provider = sshProviders.get(connectionId) // SSH — separate map +} +``` + +The daemon is installed by `setLocalPtyProvider(routedAdapter)` +(`src/main/daemon/daemon-init.ts:1031`) and adds **zero** branches to the shared +spine. The paired runtime is encapsulated behind a `PtyTransport`. SSH is neither — +it rides the local path carrying a `connectionId` tag. + +**[R] That tag leaks:** ~199 remoteness branches in main (52 in `pty.ts`, 143 in +`orca-runtime.ts`), ~191 in the renderer, plus 25 `instanceof` provider checks. +The `remote:` predicate is re-declared 5 times in the renderer. + +**[V] The transport dropped the half that makes it persistent.** +`src/main/ssh/relay-protocol.ts:34`: + +```ts +export const MessageType = { Regular: 1, KeepAlive: 9 } as const +``` + +The 13-byte frame header was copied from a well-known persistent protocol, but +only 2 of its 9 message types were kept. Dropped: `Control`, `Ack`, `Disconnect`, +`ReplayRequest`, `Pause`, `Resume`, `None`. The surviving ACK field feeds only +`unackedTimestamps` (`ssh-channel-multiplexer.ts:107,448,596`), a seq→timestamp map +read solely by the death timer. **Nothing is retained at the transport layer; +nothing is replayed.** Consequence: three layers above it each rebuilt +retain-and-replay independently (SSH credit ledger, runtime-RPC source-range +ledger, daemon batcher — the last deliberately lossy). + +**[V] Method layer is transport-agnostic.** `runtime/rpc/methods/terminal.ts` and +`git.ts` have 0 SSH references; `files.ts` has 27. + +**[V] Three durable records exist, plus a fourth copy on the host.** + +| | key | pane identity is | +| ------------------------------------- | ------------------------- | ----------------------------- | +| PTY binding (`WorkspaceSessionState`) | `(hostId, tabId, leafId)` | **the key** | +| `SshRemotePtyLease` | `(targetId, relayPtyId)` | **optional payload** | +| `SshPtyConsumerRecovery` | `targetId` | n/a (per-target resume token) | +| relay `attachIdentity` | per PTY, in-memory | compare-only | + +**[R] ~374 lines of pure lease↔binding reconciliation in `persistence.ts`**, and 9 +enumerated states where the records can disagree. Example of an independent latent +bug: absence of a lease means "restorable" in `isRestorablePtyBinding` but "not +restorable" in `hasRestorableSshRemotePtyLease` — so whether a binding survives +depends on whether a _sibling leaf_ had a binding. + +**[V] The relay holds nothing durably.** Its PTY table is +`private ptys = new Map()` (`src/relay/pty-handler.ts:356`). +No relay-side persistence of PTY state exists. `pty.serialize`/`revive` hand state +to the _client_ and take it back; `revive` requires the original PIDs to still be +alive. On PTY exit the entry is deleted with no tombstone. + +**[V] Pane identity is write-only over the wire.** `listProcesses` and `attach` +never return `paneKey`/`tabId`/`leafId`; `attach` only compares and throws +`identity mismatch`. The one RPC that returns identity (`pty.serialize`) has +**zero production callers** — test files only. + +**[V] The client's lease is consulted from a synchronous, offline, disk-only path.** +`hasRestorableSshRemotePtyLease` is called inside `setLocalWorkspaceSession` +(`src/main/persistence.ts:6465`), a synchronous `void` method invoked on every +renderer session publish including during quit (`:6287`, `:6351`). + +## 3. Alternatives already refuted + +**A. Merge the lease into the pane binding.** Blocked by two load-bearing +properties: a lease must be able to name a live shell **no pane owns** (the actual +STA-3077 field state; reattach enumerates these), and a lease **outlives** the +binding — expiry deletes the binding and then authorizes recreating a shell for 30s +(`SSH_PANE_RECOVERY_GRACE_MS`, `orca-runtime.ts:1771`). + +**B. Delete the lease; make the remote host authoritative.** Refuted. The host has +no durable state, cannot be queried for pane identity, has no notion of "recently +died, replacement authorized", and is by construction unreachable exactly when the +record is needed. Decisive: you cannot replace a synchronous local disk read during +quit with an RPC to a host that is down. + +**C. A second "authority architecture" beside the existing one** (+60,903 lines, +394 new files). Rejected earlier: it fixed none of the three root causes. + +## 4. The design under review + +### Plane 1 — data plane (adopted from a separate proposal) + +Restore a delivery guarantee at the transport, then collapse what was rebuilt above +it. Roughly: capability handshake; one delivery-identity type; one credit-ledger +implementation parameterised by unit and limit; one chunk-and-ack primitive; one +lane vocabulary; binary PTY payloads reusing the binary terminal frame Orca already +ships to mobile; prebuilt native binaries so first connect stops needing a compiler; +then teach the relay to listen on a loopback WebSocket, forward that port, and let +the _same_ client serve the SSH route — after which SSH is "launch + access" only. + +### Plane 2 — control plane (proposed here; the reviewed contribution) + +The lease answers two different questions with one record keyed for neither: + +- "which shell does this pane own?" → wants a **pane** key +- "which shells exist that nobody owns?" → wants a **pty** key + +Proposal: split by question. + +- **Ownership record**, keyed `(host, pane)` — the _same_ key local already uses. + Transport-specific payload (lease state + timestamps for SSH; nothing extra for + local). Makes "one pane, one live shell" structural rather than enforced. +- **Orphan inventory**, keyed `(host, ptyId)` — live shells with no owner. Feeds the + reattach work list and the disconnected-cleanup UI. +- The 30s recovery grant becomes a terminal state on the pane-keyed record. +- Offline consumers keep reading local disk synchronously, unchanged. + +Claimed payoff: the ~209 lines of supersession/ranking/rollback/dual-matchers exist +_only_ to bolt a pane-keyed constraint onto a pty-keyed record, and become +unnecessary. + +## 5. Historical defect, now resolved + +This defect was resolved by the shared `bindPaneShell` producer used by relay reattach and both +spawn handlers. Its mutation proof is recorded in +[S5](./new-design-goalposts.md#s5--the-superseded-pane-fence-is-live-on-the-reattach-path--proven). + +## 6. Hard constraints + +- **Cross-platform**: macOS, Linux, Windows (+ WSL). No `echo $$`/`ps` assumptions. +- **Folder workspaces**, not only git worktrees. +- **Remote wire compatibility**: clients and remote hosts update independently; + mixed versions are normal. New optional field = safe; new stream opcode must be + capability-negotiated (decoders drop unknown opcodes silently); changing what a + host publishes reaches old clients with no wire change. +- **Git 2.25 baseline** for git-touching work. +- Durable on-disk state is also read by pinned older builds in cross-version tests. +- "Unknown is not dead": disconnect, timeout, absent inventory, and retry + exhaustion must never authorize destroying a process. +- No `max-lines` lint disables. + +## 7. What the review must answer + +1. **Correctness holes.** Which cases does the Plane 2 design get wrong or fail to + cover? Specifically: multi-window / multi-device on one host; app restart while + disconnected; relay restart; host reboot; PTY exit during disconnect; two panes + racing for one shell; migration from existing on-disk state; a pane deleted while + its shell lives; the same worktree open in two windows. +2. **Simplicity.** Is there a _simpler_ design with the same correctness? Simpler + means fewer code paths and fewer concepts, not merely fewer lines. Is the + two-record split actually necessary, or is there a single-record formulation that + survives the section-3 refutations? +3. **Migration.** Re-keying a durable record needs a story for existing state, + partial writes, and rollback. +4. **Testability.** What oracle would fail if each guarantee were removed? A design + whose guarantees cannot be falsified by mutation is not finished. +5. **Is Plane 2 even the right cut?** Attack the premise that data plane and control + plane are separable here. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html new file mode 100644 index 00000000000..8718846bd98 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html @@ -0,0 +1,235 @@ + + + + + + Terminals that don't duplicate — the plain-English design + + + +
+
+

Why terminals duplicate,
and how we stop it

+

Plain English, no code. Two bugs, two fixes, then one decision for you.

+
+ +
+ The whole design in one line:
+ Keep our own notes about which terminal is whose — and never guess whether a program died. +
+ +
Setup
+

What's behind one terminal

+

When you open a terminal on a remote machine, there are three things, not one.

+
+
The three pieces⤢ zoom
+

THE PANE
the box on your screen

THE NOTE
this pane goes
with that program

THE PROGRAM
really running on
the other machine

THE PANE
the box on your screen

THE NOTE
this pane goes
with that program

THE PROGRAM
really running on
the other machine

+
+

Your network blips. The pane stays on screen. The program keeps running over there. Only the connection between them breaks.

+

So reconnecting is one job: match the notes back up to the programs. Both bugs below are that matching going wrong.

+ +
Bug one
+

The notes piled up

+

A real user report: 2 terminals became 19, then 20. Most were panes they never opened.

+
+
Why they multiplied⤢ zoom
+

you have 2 terminals

blip → reconnect

Orca writes a new note
and keeps the old one

another blip

another note.
old ones still kept.

reconnect restores EVERY note.
one pane per note.
19 panes

you have 2 terminals

blip → reconnect

Orca writes a new note
and keeps the old one

another blip

another note.
old ones still kept.

reconnect restores EVERY note.
one pane per note.
19 panes

+
+

The cause is boring and completely fixable: the notes were filed under the program's name, not the pane's. One pane could collect unlimited notes, and nothing threw the old ones away.

+ +

The fix: one slot per pane

+
+
Filed by pane instead⤢ zoom
+

after · filed by pane

pane A

its one note

before · filed by program

pane A

note 1

note 2

note 3

after · filed by pane

pane A

its one note

before · filed by program

pane A

note 1

note 2

note 3

+
+

Piling up is no longer something we prevent — there is one slot, so it cannot happen. That's the difference between a rule you enforce and a shape where the bug has nowhere to live.

+ +
+ One detail that matters +

The note is filed under a permanent serial number stamped on the pane when it's created — not under "which tab it's in". Tabs change: you can drag a pane into another tab any time. A serial number never does. File things under what doesn't move.

+
+ +
Bug two
+

"I can't find it" was read as "it died"

+

This is the worse one, because it's how you end up with two AI agents writing into the same conversation.

+
+
Drag a pane to a new tab, then reconnect⤢ zoom
+

you drag a pane
into a new tab

the other machine still has
the OLD tab written down

on reconnect it compares, sees
a difference, and replies:
no such terminal

Orca reads that as
the program died

starts a replacement,
and resumes your agent again

TWO agents writing to one
conversation. the first one
never stopped.

you drag a pane
into a new tab

the other machine still has
the OLD tab written down

on reconnect it compares, sees
a difference, and replies:
no such terminal

Orca reads that as
the program died

starts a replacement,
and resumes your agent again

TWO agents writing to one
conversation. the first one
never stopped.

+
+ +
+ The trick in it +

The other machine did find your program. Comparing is how it noticed the tab was different. But it answered "not found" — so a message meaning "it's alive and I'm being fussy" looked exactly like "it's gone."

+
+ +

Fix, part one: stop comparing tabs

+

Moving a pane between tabs is a normal thing you're allowed to do, so it must never make a terminal unreachable. That comparison is removed.

+

It was also failing at its real job anyway. It existed to catch a different problem — the remote machine reusing an old ID for a brand-new program, like an apartment number being handed to a new tenant. It never caught that. So we check the tenant instead of the apartment number: catches the real case, stops rejecting panes that merely moved.

+ +

Fix, part two: never guess about death

+

A replacement terminal starts only on actual proof the old program is gone.

+
+
What counts as proof⤢ zoom
+

did the program die?

we watched it exit

the same helper that started it
says it's gone

can't reach the machine

it timed out

missing from a list

confusing error message

PROOF
start a replacement

NOT PROOF
leave it running.
ask the user.

did the program die?

we watched it exit

the same helper that started it
says it's gone

can't reach the machine

it timed out

missing from a list

confusing error message

PROOF
start a replacement

NOT PROOF
leave it running.
ask the user.

+
+

The second proof is the clever one. The helper on the remote machine keeps its list of running programs only in memory. So if that same helper — same process, not a restarted one — says a program isn't in its list, it must have exited. There is nowhere else it could be.

+

And if the helper restarted, it remembers nothing, so we say we don't know. We never treat a fresh helper's empty list as news that your programs died.

+ +
Result
+

What you'd actually notice

+ + + + + + + + +
SituationTodayAfter
Network drops and comes backTerminals multiply; the remote machine fills with junkYour terminals come back. Nothing invented.
You drag a pane to a new tabA second agent resumes onto the same conversationJust works.
Orca genuinely can't tell if a program survivedSilently starts a new onePane says "disconnected", with two buttons
Leftover programs on a remote machineInvisible, pile up foreverListed with folder and name — you choose what to close
+ +
+
The one visible change⤢ zoom
+

Orca isn't sure whether
your program survived

TODAY
quietly starts a new one.
you may not notice until
there are two agents.

AFTER
pane shows 'disconnected'
[ try again ] [ new terminal ]

Orca isn't sure whether
your program survived

TODAY
quietly starts a new one.
you may not notice until
there are two agents.

AFTER
pane shows 'disconnected'
[ try again ] [ new terminal ]

+
+ +
+ This is the decision I need from you +

Today, uncertainty gets resolved silently and wrongly. After this, uncertainty becomes visible — a disconnected pane with two buttons.

+

Slightly more friction in a rare case, in exchange for never resuming your agent twice. I think it's clearly right, but it changes what people see, so it's your call.

+
+ +
Plan
+

The order we'd build it

+

Six steps. Each ships on its own and can be undone on its own.

+ + + + + + + + + + +
StepWhat it does
1Stop the tab comparison killing live terminalsFixes the double-agent bug. Small, self-contained, helps immediately.
2Stop faking "the program exited"Today a failed reconnect tells the pane the program exited — a claim we can't back up. Ships with the two-button UI.
3Write the notes in one placeThey go to two different places today, which makes the cleanup silently do nothing. This is the actual cause of 2 → 19.
4One piece of code writes the notesTwo paths write them and a third forgets to — which is why an existing safety check never runs during reconnect.
5Check whether an old recovery path ever runsTwo reviewers believe it's unreachable. Measure before building on it.
6The proof ruleOnly if step 5 shows it's needed.
+ +
+

The five words I couldn't avoid

+ + + + + + + + +
PaneOne terminal box on your screen.
ProgramThe shell or agent actually running — often on another machine.
NoteOrca's record that a pane goes with a program. Saved to disk, so it survives a restart.
HelperThe small program Orca installs on a remote machine to run terminals there.
OrphanA program still running that no pane claims. We list them; we never close them for you.
+
+ +
+ Why this is smaller, not bigger +

It removes roughly 900–1,250 lines and adds roughly 450–700. Most of what goes away exists only to patch up disagreements between notes filed different ways. Once there's one note per pane, that code has nothing left to do.

+
+ +
+ The detailed versions: counsel-design.html — how this was reviewed and what it cost. + report.html — evidence for the work already written. + Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
×
scroll to zoom · drag to pan · Esc to close
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md new file mode 100644 index 00000000000..d6b474c93a8 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md @@ -0,0 +1,256 @@ +FINAL DESIGN — one leaf-keyed ownership record; pane identity deleted from the attach path; death proved only by the relay process that spawned the shell + +===================================================================== +D0. SCOPE AND THE ONE RULE EVERYTHING FOLLOWS FROM +===================================================================== +Problem (from the brief): SSH reconnect grows pane cardinality (2 -> 19 -> 20, STA-3077); a coding agent gets resumed twice into one transcript (RC2); one PTY's failure can drop a whole host connection (RC3, out of scope — Plane 1). + +One rule, applied without exception: + +Ownership ("which shell does this pane own?") is DURABLE, CLIENT-LOCAL, and read +synchronously from disk while offline. Evidence about a shell's FATE is +NON-DURABLE, HOST-LOCAL, and must be REPORTED by the host, never INFERRED by the +client from timing, from the absence of an entry, or from the shape of an error +string. + +Every defect found across three review rounds is one violation of the second clause. The design's job is to remove all of them, not to add an authority layer. + +SCOPE DECLARATION (grok, low): the record is host-agnostic (`hostId` is a field). The _death rule_'s row 2 is relay-specific. Local and daemon panes reach `terminated` only through row 1 (an exit observed in-process, which is how they already work) and never through row 2. That is stated rather than left implicit. + +===================================================================== +D1. THE DURABLE RECORD +===================================================================== + +```ts +/** Main-owned. TOP-LEVEL in PersistedState — never inside WorkspaceSessionState. */ +type PaneShellOwnership = { + leafId: string // PRIMARY KEY. Terminal-layout leaf UUID. + hostId: ExecutionHostId // FIELD: 'local' | 'ssh:' | 'runtime:' + worktreeId: string // FIELD. Opaque FULL string incl. any `::workspace:` suffix. + ptyId: string // RELAY-NATIVE form for SSH (toRelaySshPtyId). One namespace, always. + incarnationId: string // REQUIRED. Relay mints one per pty (relay/pty-handler.ts:1525). + relayInstanceId?: string // The relay PROCESS that owned it, as reported at bind time. + state: 'attached' | 'detached' | 'terminated' + createdAt: number + updatedAt: number + lastAttachedAt?: number + lastDetachedAt?: number +} +// Storage: PersistedState.paneShellOwnershipByLeafId: Record +``` + +Note what is NOT here: no `tabId`, no `paneKey`, no `hostEpoch`, no `resumed`, no second (pty-keyed) record. + +WHY `leafId` AND NOT `paneKey`. `makePaneKey(tabId, leafId)` returns `${tabId}:${leafId}` and the repo states at src/shared/stable-pane-id.ts:47-49 that "only the leaf UUID is remint-stable pane identity (the tab half changes on pane break-out)" — which is why `isEquivalentPaneKey` exists at :50-57. The mutation is a shipping gesture: `detachTerminalPaneToTab` (src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.ts:245-310) moves a LIVE pane and its ptyId into a new tab via `createTab(..., initialPtyId)` while explicitly not killing the PTY (:275-276). `leafId` is a UUID and is never remapped once it is one (src/main/persistence.ts:1858-1872 skips ids already passing `isTerminalLeafId`). Keying by the leaf deletes the transfer problem instead of managing it. + +WHY NO `tabId` AT ALL (this is the round-3 change). A stored tabId is either frozen (and then it names a tab the leaf no longer lives in) or updated (and then it desynchronizes from the relay's frozen copy). Both are live production failures — see D2. The tab is LOCATION, derived from the live layout at the moment it is needed; it is never stored and never trusted. + +WHY TOP-LEVEL. `session:set` / `session:set-sync` (src/main/ipc/session.ts:13, :31) accept a whole `WorkspaceSessionState` from a renderer with no revision or fencing token; the clobber guards at src/main/persistence.ts:6379, :6384-6476 exist only because of that. `sshRemotePtyLeases` is already top-level. The KEY is shared with the layout; the STORAGE LOCATION is not. + +UNIQUENESS. At most one non-terminal record may name a given `(hostId, ptyId, incarnationId)`, enforced on write through a reverse index. Not ptyId alone: leases store the relay-native id (`getRelayPtyIdForSshLeaseStorage` -> `toRelaySshPtyId` strips the connection scope, src/main/persistence.ts:6641-6643, src/shared/ssh-pty-id.ts:58-67) and relay ids are `pty-${this.nextId++}` with `nextId = 1` per relay PROCESS (src/relay/pty-handler.ts:356-357, :1440-1443), so `pty-1` recurs on every host after every relay restart. Round 1's rejection ("a cross-target collision on ptyId is impossible") is withdrawn; it is guaranteed, not impossible. + +===================================================================== +D2. DELETE PANE IDENTITY FROM THE ATTACH PATH (the largest round-3 change) +===================================================================== +Both reviewers independently attacked the attach-time identity comparison. Verified, and it is worse than either stated: the apparatus is simultaneously over-strict and under-strict, and it is a live duplicate-agent bug today. + +VERIFIED FACTS + +- The relay compares frozen strings: `attachIdentityMismatches` (src/relay/pty-handler.ts:334-339) does full-string `expected.paneKey !== managed.paneKey || expected.tabId !== managed.tabId`; `attachIdentity` is captured once at spawn from `ORCA_PANE_KEY`/`ORCA_TAB_ID` (:1512-1517, :1446). On mismatch it throws `PTY "" not found (identity mismatch)` (:1616-1626). +- `isSshPtyNotFoundError` is a message regex `/PTY ".+" not found/i` (src/main/providers/ssh-pty-errors.ts:12-15), so it is TRUE for the mismatch string. `reattachSshPtySession` re-emits it as `SSH_SESSION_EXPIRED: SSH_PTY_IDENTITY_MISMATCH` (src/main/providers/ssh-pty-session-reattach.ts:218-226). +- The renderer's `isProvenSshSessionGoneError` returns TRUE for anything containing `SSH_SESSION_EXPIRED` and for the raw not-found regex, and IGNORES the mismatch marker (src/renderer/src/components/terminal-pane/reattach-failure-classification.ts:24-30). Its two call sites then clear the binding and call `startFreshColdRestoreAgentResume` (pty-connection.ts:8839-8847, :9086-9097) — a second shell and a SECOND AGENT RESUME while the original shell keeps running. That is RC2, on a live process, reachable from a shipped gesture. +- The spawn-path reattach builds the expected identity from the CURRENT options (`args.options.paneKey ?? env.ORCA_PANE_KEY`, ssh-pty-session-reattach.ts:66-68), i.e. the current tab — so after a detach it mismatches. The connect-path reattach builds it from the lease's stored tabId (`expectedIdentityForLease`, src/main/ssh/ssh-relay-session.ts:190-210, consumed at :2201-2210), which is never re-keyed on detach — so it matches, attach succeeds, and then `persistPtyBinding(tabId = old tab, mayCreate:false)` refuses because the leaf is no longer in that tab's layout (src/main/persistence.ts:6821-6877 sets `terminalMembershipChanged` and returns false), leaving the shell running unbound (ssh-relay-session.ts:2513-2521). Grok's "scissors" is confirmed on both blades. +- The apparatus does not even do its own stated job ("generation resets can reuse PTY IDs; reject conflicting identities", pty-handler.ts:1615): after a relay restart that recycles `pty-3` for the SAME pane, paneKey and tabId both match and a different shell is accepted. + +RESOLUTION — delete the comparison; do not replace it with another comparison. + +1. Relay: delete `attachIdentity`, `attachIdentityMismatches`, the `params.expectedPaneKey`/`expectedTabId` handling and the `(identity mismatch)` throw. `ManagedPty.paneKey`/`tabId` STAY — they are documented as separate from attach identity (pty-handler.ts:139-143) and are used by the exit listener (:765) and, newly, by listProcesses reporting (D4). paneKey stops being an authorization INPUT and becomes a reported HINT. +2. Main: delete `ExpectedPtyIdentity`, `expectedIdentityForLease`, `expectedIdentityByPtyId`, the `expectedIdentity` parameter threaded through `attachPtyWithRetry`, `isSshPtyIdentityMismatchError`, `SSH_PTY_IDENTITY_MISMATCH_ERROR`, the marker append, and the mismatch branch at ssh-relay-session.ts:2628-2634. +3. Renderer: `isProvenSshSessionGoneError` keeps only proofs. (After step 1/2 the mismatch string ceases to exist; the renderer fix ships FIRST and independently, because it must protect users running against relays that have not been redeployed.) +4. Recycled-id rejection moves to two places that already hold exact data: + (a) PROCESS GATE, once per connection: if the owner grant reports a `relayInstanceId` different from the one recorded on a record, that record's ptyId is not attachable at all. One comparison covers every pty on the host. Within a single relay process ids never recycle (`nextId` is monotonic), so this is complete against new relays. + (b) PER-ATTACH GUARD, zero wire change: EVERY `pty.attach` success path already returns `incarnationId` (src/relay/pty-handler.ts:1646, :1653, :1668, :1676), and the client already parses and remembers it (`requestSshPtyAttach` -> `rememberPtyIncarnation`, ssh-pty-session-reattach.ts:74-100). Compare it to `record.incarnationId`; on mismatch, throw — the existing `activationLease.rollback()` already runs on a thrown error — and classify `detached`, never death. `RecentPtyOutputBuffer.read()` is explicitly non-mutating (src/main/runtime/recent-pty-output-buffer.ts:75-85), so abandoning a wrongly-attached shell does not consume its replay. + This is strictly stronger than the frozen-string check (a per-pty UUID vs a pane label), needs NO new request field, and works against OLD relays — which a new `expectedIncarnationId` request parameter would not. +5. The reattach BIND resolves the tab from the live layout: find the tabId whose `terminalLayoutsByTabId[*].root` contains this `leafId`. If no layout contains it, refuse exactly as today (leave the shell running unbound, "no durable pane owns this"). `mayCreate:false`'s guarantee is preserved; its dependence on a stored tabId is deleted. + +ONE IDENTITY RULE, EVERYWHERE: the leaf is identity; the tab is derived location; the incarnation is shell identity; the relay instance is process identity. There is no fourth. + +===================================================================== +D3. THE DEATH RULE — two rows, one new field +===================================================================== +`resumed` and `hostEpoch` are deleted and must not be built. + +- `resumed: replaces !== null` (src/shared/pty-consumer-session.ts:275); `expireOwner()` nulls the incumbent after `PTY_CONSUMER_OWNER_GRACE_MS = 30_000` (:281-287, contract :2), after which admission takes `if (!current) return this.newOwner(...)` (:215-216) and `resumed` is false regardless of proof. Unreachable for quit, sleep, overnight, reboot — the normal case. Its own contract comment says it means "the client's checkpoints for the previous claim no longer apply" (pty-consumer-session-contract.ts:49-52): a delivery-checkpoint flag, not liveness. +- `hostEpoch = Date.now() - os.uptime()*1000` under exact equality is a reconstructed estimate; second-granularity uptime, NTP steps and suspend/resume make it differ on an unrebooted host, and "differs" was the grant-YES row — it fails OPEN toward respawn. Repo precedent settles it: `readCurrentDaemonReadyIdentity` reads `/proc/sys/kernel/random/boot_id` on Linux only and returns bare `{ startedAtMs }` elsewhere (src/main/daemon/daemon-ready-identity.ts:11-13). + +THE ONE NEW GRANT FIELD: `relayInstanceId?: string` on `PtyConsumerSessionGrant` (src/shared/pty-consumer-session-contract.ts:43-59) — a `randomUUID()` minted once at relay process start (`randomUUID` is already imported at relay/pty-handler.ts:6). It is genuinely new information: the grant's existing `serverBuildId` is `launchVersion` (src/relay/relay.ts:694-696), a build string that is identical across restarts of the same build. + +| Condition | Transition | Respawn grant | +| ---------------------------------------------------------------------------------- | ------------ | -------------------- | +| PTY exit observed on a live attached stream | `terminated` | yes | +| attach returns not-found AND the grant's `relayInstanceId` EQUALS the recorded one | `terminated` | yes (subject to E-1) | +| attach returns an incarnation different from the recorded one | `detached` | no | +| `relayInstanceId` absent on either side, or differing | `detached` | no | +| any other error (today's `restoreRequired` path, ssh-relay-session.ts:2612-2620) | `detached` | no | + +Row 2 is sound and UNBOUNDED IN TIME: `this.ptys` is the relay process's only store (pty-handler.ts:356) and entries are deleted on exit, so "the same process that minted this pty no longer has it" means it exited. It is sound on Windows too, because the relay OBSERVED the exit — unlike any rule that reasons from a relay restart. A restarted relay reports a different id and we correctly claim no knowledge. + +SHIP GATE (grok H4): row 2 grants a respawn on a not-found detected by a message regex. One shared normalization helper (`toRelaySshPtyId`) must be applied on every attach / mark / record / comparison path, and the record stores exactly one namespace (D1), so a namespace mismatch cannot manufacture a false not-found. Row-2 grant-yes does not ship until E-1 is green. + +STOP FABRICATING THE EXIT. `handlePtyReattachFailure` currently sends `pty:exit { code: -1 }` (ssh-relay-session.ts:2643-2647) plus `clearProviderPtyState` + `deletePtyOwnership` + `markSshRemotePtyLease('expired')` on a plain not-found. On every `detached` row this is a lie about a process the client knows nothing about. Remove the whole block and route the unknown case into the branch that ALREADY EXISTS three lines above it: `pending.restoreRequired = 'reattachAttemptsExhausted'` + `wakeRecovery` (:2612-2620), which is non-destructive by construction. This is a branch COLLAPSE, not a new signal — which is also the answer to the objection that removing only the send turns a wrong signal into no signal. + +===================================================================== +D4. ORPHANS — a connect-time projection, never an authority +===================================================================== + +``` +orphans(host) = successful listProcesses(host) + MINUS { (ptyId, incarnationId) named by a non-terminal record on this host } +``` + +Computed at connect/reattach, held in memory for the connection's life, discarded on disconnect. No durability, no migration, no rollback. + +WHAT `listProcesses` IS: it iterates only `this.ptys` (pty-handler.ts:1894-1911) — the current relay process's Map. No durability, no OS scan. It is therefore (i) the cleanup-UI source and (ii) a SUPPLEMENT to the reattach work list, never the sole discovery authority. The work list stays `getPtyIdsForConnection(target) UNION` the non-terminal records (today's `leasedPtyIds` shape, ssh-relay-session.ts:2211-2219). + +ABSENCE IS NEVER AUTHORITATIVE. The repo already adjudicated this with an issue number: `pty:listSessions` documents "Absence is authoritative only from a provider that serializes claims — otherwise it is 'unknown', never 'absent' (#8459)" (src/main/ipc/pty.ts:7343-7347), and `SshPtyProvider.hasPty` returns `null` before a completed listing "a miss there is ignorance about the host, not a dead PTY" (src/main/providers/ssh-pty-provider.ts:308-312). The only aggregating caller launders every remote failure with `provider.listProcesses().catch(() => [])` (ipc/pty.ts:7333), and `PtyProcessListAdmission.admit` throws on capacity and on `agent_session_ownership_unknown` (src/main/providers/pty-process-list-admission.ts:60-99). So: a rejected or truncated listing yields orphans = UNKNOWN (empty UI, no claims, no retirement), never orphans = everything. + +THE ONE NEW RESPONSE FIELD: `PtyProcessSummary` gains optional `paneKey`. The relay already holds it (`ManagedPty.paneKey`, pty-handler.ts:1446, :139-140) and omits it from the summary (:1899-1908). Additive optional response field — safe under docs/reference/remote-wire-compatibility.md, no opcode, no negotiation. It does NOT reach the client for free: `PtyProcessInfo` has no `paneKey` (src/main/providers/pty-process-info.ts:4-16) and `admit` REBUILDS an allowlist (pty-process-list-admission.ts:106-115). Three edits — relay summary, `PtyProcessInfo`, allowlist — and the oracle asserts survival THROUGH `admit`. `incarnationId` and `worktreeId` already traverse that whole chain, which is what makes the incarnation-scoped subtraction free. + +ADOPTION matches `parsePaneKey(reported.paneKey)?.leafId` — never the full string, because the relay's copy is frozen at spawn and names a tab the leaf may have left. Because the record is leaf-keyed this is a direct lookup, not fuzzy equivalence; `isEquivalentPaneKey` is not imported here. Adopt is a CAS: write `leafId -> (ptyId, incarnationId)` only if no non-terminal record exists for that leaf AND none on this host names that `(ptyId, incarnationId)`. Exactly one of two concurrent adopts wins; the loser gets a typed refusal. + +KILL POLICY (replaces the "owned elsewhere" classification, which is DELETED). Both reviewers attacked `worktreeId`-based classification from opposite directions and both are right, because `worktreeId = ::` and `repoId` is a client-local `randomUUID()` minted at repo add (src/shared/worktree-id.ts:15-28; src/main/runtime/orca-runtime.ts:18729). So "unknown worktreeId" means "not in this install's current state file": it wrongly EXEMPTS the user's own shells after a repo re-add, profile transfer or state reset, and it wrongly INCLUDES another device's shells whenever the two installs share state. One rule replaces it: the orphan list never auto-kills, never bulk-kills, and never kills as a side effect of any reattach or cleanup pass. Kill is an explicit per-shell user action, with cwd, title, worktreeId and reported paneKey shown. No classifier, no new field, and it is safe in both directions. + +OLD-RELAY DEGRADATION: a relay that omits `paneKey` yields unclassified orphans that are listed and never killed — exactly today's behavior. + +===================================================================== +D5. RECORD LIFETIME — three exits, none of them "absent from a list" +===================================================================== + +1. RETIRE (delete). The record reached `terminated` (row 1 or row 2 of D3, or explicit user close via `retireExitedPty`, ssh-relay-session.ts:2153-2172). Terminal records are garbage-collected after a bounded retention (long enough to serve any surviving grant window) so they cannot mask a recycled `pty-1`. There is NO retire-on-absence clause: the case it was written for (shell exited during a disconnect, relay still up) is already covered by row 2 — the attach itself proves the exit — so deleting the clause costs nothing and removes a direct contradiction with #8459. +2. RELEASE (keep the shell, drop the claim). Fired ONLY from the explicit pane-close and tab-delete code paths — never from "this publish did not contain the leaf". Detach is not atomic across layouts: `createTab` installs an empty layout, then `setTabLayout(sourceTabId, ...)` removes the leaf, then `setTabLayout(tab.id, ...)` adds it (terminal-pane-tab-detach.ts:298-300), so between the last two no layout contains the leafId, and any `session:set` flush in that window would look like a durable removal. Released records are cleared to a non-claiming state: the shell becomes orphan-eligible (visible, adoptable) and is deliberately NOT `terminated`, so it can never authorize a respawn. +3. EXPLICIT USER CLOSE -> `terminated`, and the record dies with the shell. + +RESIDUAL, STATED HONESTLY: a shell that died while the relay ALSO restarted leaves a `detached` record with no proof available. It stays restorable until the user acts. That is the conservative direction and it is bounded by the affordance below, not by a timer. + +PRODUCT AFFORDANCE, REQUIRED IN THE SAME PR. A `detached` pane renders as disconnected with two explicit actions — "reattach (retry)" and "start a new shell" (the latter retires the record). Nothing infers death, nothing auto-spawns. Per AGENTS.md this must follow docs/STYLEGUIDE.md. COPY CONSTRAINT: the UI must never assert that a shell is dead. The claim "after a relay crash the shells are almost certainly dead" is REMOVED from this design: it rests on POSIX master-fd close SIGHUPing the foreground group, and on Windows the relay is deliberately launched through WMI because "Windows sshd kills the exec channel's process tree on close" (src/main/ssh/ssh-relay-deploy.ts:1805), with ConPTY and no SIGHUP. No rule in this design concludes anything from a relay restart, so the design is correct either way; only the copy had to change. + +===================================================================== +D6. HISTORICAL — one partition per (target, pane). SUPERSEDED. +===================================================================== +This migration proposal was rejected. The shipped containment keeps the existing persistence +planes and gives the pane binding one local home; see [S4](./new-design-goalposts.md#s4--one-partition-per-target-pane--proven). + +SSH pane bindings live in two partitions today. Readers and writers that consult ONLY `ssh:`: `resolvePersistedStablePaneOwner` (src/main/ipc/pty.ts:676-677), `retirePersistedStablePaneOwner` (:760-761, :783), the CAS write `persistPtyBinding(..., expectedBinding)` (:833), and the two spawn upserts (:5108, :6493). Against that, the relay reattach write has no hostId (ssh-relay-session.ts:2500-2511 -> `resolveHostId(undefined)` -> `LOCAL_EXECUTION_HOST_ID`, persistence.ts:6243-6246) and the renderer keeps SSH worktrees in `local` deliberately (workspace-session-host-persistence.ts:167-173). `durablyBoundPtyIdForPane` hedges ssh-first-then-local (persistence.ts:7269-7277), so `supersedeSiblingLeasesForPane` early-returns when the bound pty differs (:7229-7232) — supersession silently no-ops and both leases stay live. That is the STA-3077 mechanism itself. + +FIX: one accessor `sessionForSshPaneBindings(connectionId): WorkspaceSessionState` returning the `local` partition, routed through by EVERY reader and EVERY writer above, flipped in a single atomic PR — not five write-site edits. `local` wins because the only publisher of pane membership writes there and `mayCreate:false` is evaluated there. A one-time fold moves `ssh:.terminalLayoutsByTabId[*].ptyIdsByLeafId` and `tabsByWorktree[*].ptyId` into `local`, preferring `local` on conflict; the loser is NOT demoted to orphan-candidate by ptyId alone (recycled ids collide) — it carries `incarnationId` or is dropped. + +===================================================================== +D7. HISTORICAL — one bind producer. RESOLVED. +===================================================================== +The shared `bindPaneShell` producer now serves relay reattach and both spawn handlers; see +[S5](./new-design-goalposts.md#s5--the-superseded-pane-fence-is-live-on-the-reattach-path--proven). + +`rememberPaneKeyForPty` (ipc/pty.ts:517-525) has two callers, both spawn (:5222, :6668). `restoreReattachedPtyRuntime` calls `runtime.registerPty` instead (ssh-relay-session.ts:2523), and `isSupersededPtyId` returns `false` for an unrecorded id by design (:284-296) — so the shipped superseded-PTY fence is INERT on the path it was built for. Collapse to one `bindPaneShell({ hostId, worktreeId, leafId, ptyId, incarnationId, relayInstanceId })` that, in one call, (a) resolves the current tab containing the leaf (refusing if none), (b) writes the durable record, (c) writes the in-memory `ptyPaneKey`/`paneKeyPtyId` fence maps using the paneKey composed from the CURRENT tab, (d) calls `registerPty`. All three paths call it and nothing else. + +===================================================================== +D8. COLLAPSE THE RESTORABLE FORK — and the offline/synchronous path +===================================================================== +`setLocalWorkspaceSession` forks on whether the INCOMING layout map is empty (persistence.ts:6450-6473): empty -> `isRestorablePtyBinding` (absence of a lease implies restorable, :6620-6631); partial -> `hasRestorableSshRemotePtyLease` (requires a live lease, :6670-6685). They disagree on the absent-record case, so whether a binding survives depends on whether a SIBLING LEAF had one. + +One predicate replaces both: RESTORABLE IFF `paneShellOwnershipByLeafId[leafId]` exists with state `attached` or `detached`. + +OFFLINE / SYNCHRONOUS ACCESS PATH (refutation B, honored exactly). `setLocalWorkspaceSession` is a synchronous `void` method invoked on every renderer publish including during quit (persistence.ts:6465, :6287, :6351). The loop at :6448-6462 already iterates `priorLayout.ptyIdsByLeafId` and therefore already holds a `leafId`, so the new predicate is a DIRECT map lookup on a top-level, main-owned, in-memory-mirrored disk record: no key composition, no lease-array scan, no `incomingHasAnyBinding` branch, no mux, no RPC, no host. It works with the host down and during quit — which is precisely what refutation B says cannot be replaced by asking the host. + +===================================================================== +D9. THE REATTACH ALGORITHM (SSH, on connect) +===================================================================== + +1. Obtain the consumer owner grant. Read `grant.relayInstanceId` (may be absent — old relay). +2. Work list = `getPtyIdsForConnection(target)` UNION `{ r.ptyId | r.hostId === 'ssh:'+target and r.state !== 'terminated' }`, every id normalized through `toRelaySshPtyId`. +3. PROCESS GATE. For each record: if both `grant.relayInstanceId` and `r.relayInstanceId` are present and differ, do not attach by that id. Set `detached`. Its shell, if any, is rediscoverable only through step 5. +4. For each remaining id, call `pty.attach { id, cols, rows }` — with NO expectedPaneKey and NO expectedTabId. + 4a. SUCCESS -> if a record exists and `response.incarnationId !== r.incarnationId`, throw (activation lease rolls back), set `detached`, no grant, no exit. Otherwise call `bindPaneShell(...)`: resolve the current tab containing `leafId`; if none, log and leave the shell running unbound (today's refusal, preserved); else write record (`attached`, `relayInstanceId = grant.relayInstanceId`), fence maps, `registerPty`. + 4b. NOT-FOUND -> if `grant.relayInstanceId` is present and equals `r.relayInstanceId`: `terminated`, retire, grant authorized (gated on E-1). Otherwise: `detached`, no grant, and route into the existing `restoreRequired = 'reattachAttemptsExhausted'` branch. No `pty:exit -1`, no `clearProviderPtyState`, no `deletePtyOwnership`. + 4c. ANY OTHER ERROR -> `detached` via the same existing branch. +5. Call `listProcesses` best-effort. If it REJECTS or admission truncates it: orphans = unknown; do nothing. If it RESOLVES: orphans = returned MINUS `(ptyId, incarnationId)` named by non-terminal records on this host. For each orphan with a reported `paneKey`, look up `parsePaneKey(paneKey)?.leafId`; if that leaf exists in the live layout and has no non-terminal record, CAS-adopt it via `bindPaneShell`. The rest are listed in the cleanup UI, never auto-killed. + +===================================================================== +D10. MIGRATION, SEQUENCING, ROLLBACK +===================================================================== +ORDER — each its own PR, each independently revertible: + +| | Step | Contents | +| --- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A | Identity-mismatch is not death | Renderer: `isProvenSshSessionGoneError` stops treating the mismatch marker as proof. Relay+main: delete the whole attach-identity apparatus (D2). Pure bug fix, no design dependency, ships FIRST; the renderer half must ship even before relays are redeployed. | +| E-0 | Stop lying about exits | Collapse the plain-not-found branch of `handlePtyReattachFailure` into the existing non-destructive `restoreRequired` branch (D3). Ships with, or immediately followed by, the D5 disconnected-pane affordance. | +| P | Single partition | D6: one accessor for every reader and writer, plus the fold migration, atomically. | +| F | Single bind producer | D7: `bindPaneShell`; makes the shipped fence live on the reattach path; introduces tab-from-layout resolution. | +| E-1 | Prove the grant executes | Production-id-shape oracle for `recoverTerminalPane`; record the actual behavior. | +| E-2 | Death rule | `relayInstanceId` on the grant + the D3 table. Conditional on E-1. | +| W | Projection | `paneKey` through relay -> `PtyProcessInfo` -> admission allowlist; incarnation-scoped subtraction; explicit-only kill (D4). | +| K | Re-key | The leaf-keyed record + D8 + D5 lifetime + `reassignSshTargetId` field rewrite + porting all lease readers. | + +E-1 CONTEXT (why E-2 is conditional). The chain `not-found -> expired -> 30s grant -> createTerminal` has one spawn-authorizing caller, `recoverTerminalPane` (src/main/runtime/orca-runtime.ts:16449-16490), gated on `getRecentExpiredSshLease(...)`, whose comparison is raw `lease.ptyId === ptyId` with no normalization (:6310). `lease.ptyId` is relay-native (persistence.ts:7184) while the graph's `pty.ptyId` is app-form `ssh:@@pty-N` (ssh-relay-session.ts:2523), so for a real SSH pane they can never be equal, and the branch also cannot be reached by a local pane (it requires an SSH lease). The covering test seeds both sides as `'pty-expired'` and registers with a NULL connectionId (orca-runtime.test.ts:1350-1370, :2932-2940) — a shape production cannot produce. Do not build grant-arbitration machinery before E-1 shows the branch executing. + +FORWARD MIGRATION AT K, per `SshRemotePtyLease`: + +- has `leafId` -> one record. `state`, `createdAt`, `updatedAt` copied VERBATIM (any surviving grace window reads `updatedAt`). `worktreeId` copied as an OPAQUE FULL STRING — the `::workspace:` folder-workspace suffix must survive, since matching is full-string equality (persistence.ts:7239, orca-runtime.ts:6308) and issue #12474 was a comparator that stripped exactly that. `ptyId` stays relay-native. `tabId` is DROPPED (D1). `relayInstanceId` absent -> the record falls to `detached` on first reconnect with no ambiguity to resolve. `incarnationId` absent on a legacy row -> treat as adoption-eligible rather than as a subtracting claim, so it cannot mask a recycled id. +- no `leafId` -> no record; recovered at first reconnect by the leaf-half `paneKey` match, or listed as an orphan. +- ALSO synthesize a record from any live layout binding (`ptyIdsByLeafId`) whose ptyId has no lease, so a bound-but-unleased pane is not silently demoted. +- Sequence AFTER the legacy numeric-leaf remap (persistence.ts:1858-1872) so every record keys on a UUID. + +PORT EVERY LEASE READER — the "write-only legacy projection" claim was FALSE and is corrected. Verified production readers of `sshRemotePtyLeases` outside persistence.ts: + +- `getRecentExpiredSshLease` (orca-runtime.ts:6295-6316) with `ptyId` UNDEFINED, feeding `hasRecentExpiredSshLeasePane` (:6318-6323) -> the headless-mobile terminal-tab filter (:5290-5299 region, the `onlyRuntimeOwnedTerminals` filter) and `hasLiveOrPersistedServeOrSshOwnedPtyBinding` (:6355-6360). These are a live VISIBILITY signal for paired mobile/HUB clients and are NOT killed by the namespace bug, because they pass `ptyId` undefined. Port: "a non-terminal record names this leaf" replaces "an `expired` lease within 30s". Visibility becomes bounded by the D5 lifetime rule instead of a timer; that behavior change is deliberate and must be covered by an oracle. +- `ssh:terminateSessions` (src/main/ipc/ssh.ts:1282-1300) enumerates leases and distinguishes owned from expired (#2626). This is the EXISTING kill affordance; the D4 orphan UI FEEDS it rather than duplicating it. It is re-expressed over records: non-terminal records plus projection entries, with kill always explicit (D4). +- `ssh:resetRelay` (ipc/ssh.ts:1370-1385) marks every live lease `expired` after force-killing the relay. Ported to: mark records `detached` (never `terminated` — the client killed the relay, which proves nothing about the shells). BEHAVIOR CHANGE, flagged for the owner: a user-initiated relay reset stops silently re-showing panes for 30s and instead shows the disconnected affordance. +- `ssh-relay-session.ts:1914, :2197` internal lease reads -> record reads. + +CROSS-VERSION READS. Pinned older builds read `sshRemotePtyLeases`. For ONE release the new writer emits a derived legacy projection of the array. It must emit RELAY-NATIVE ptyIds via `toRelaySshPtyId` — leaking app-form ids gives pinned builds leases matching nothing, a silent no-op that looks like a passing test. Because the record already stores the relay-native form, this is a copy, not a conversion. It carries no `tabId`-dependent semantics, so old builds that call `expectedIdentityForLease` simply get `null` and attach without an identity — which, after step A, is the correct behavior anyway. + +PARTIAL WRITE / ROLLBACK. Reuse the existing discipline (persistence.ts:7346-7360): snapshot, mutate in memory, `flushOrThrow`, undo on failure. One record, so this is a single-record transaction. Rollback of K: the legacy projection lets an older build boot on new state; the one-way loss is leafId-less rows, which the old build also could not act on. + +`reassignSshTargetId` (persistence.ts:7055-7086) today re-keys partitions, rewrites lease `targetId`s and drops recoveries. Because `hostId` is a FIELD and not part of the key, it only rewrites a field in place plus the reverse index; no re-keying. + +===================================================================== +D11. ORACLES — each must redden under a stated mutation +===================================================================== + +1. DETACH-TO-TAB OVER SSH, THROUGH THE REAL RELAY COMPARISON. Bind an SSH pane in tab A leaf L, run `detachTerminalPaneToTab` to tab B, disconnect, reconnect. Assert: exactly one shell on the host, no second agent resume, the pane reattaches and the ptyId lands in tab B's layout. THIS MUST BE RED BEFORE STEP A. If it is green today, the harness stubbed the relay comparison or omitted `ORCA_PANE_KEY`/`ORCA_TAB_ID` from the spawn env, and the whole detach oracle family is measuring nothing. Gate the program on it going red first. +2. Identity mismatch is not proof of death. Assert `isProvenSshSessionGoneError` is false for the mismatch string, no record reaches `terminated`, no grant. Mutation: restore `.includes(SSH_SESSION_EXPIRED)` as the sole predicate (reattach-failure-classification.ts:29). +3. Wrong-incarnation attach is abandoned, not adopted and not fatal. Relay returns a different `incarnationId` than recorded: assert no bind, no stream, `detached`, no grant, activation lease rolled back. Mutation: skip the response check. +4. Same-instance not-found is proof; different or absent is not. Two runs, recorded `relayInstanceId` equal -> `terminated` + grant; differing/absent -> `detached` + no grant. Mutation: flip the comparison. +5. Failed listing is not an empty listing. Make `listProcesses` reject, and separately make `admit` throw `agent_session_ownership_unknown`: assert zero records retired and zero orphans reported. Mutation: replace the failure with a resolved `[]` — must redden, proving the implementation does not reuse the `.catch(() => [])` shape at ipc/pty.ts:7333. +6. No fabricated exit on unknown. Drive `handlePtyReattachFailure` with a not-found and a differing `relayInstanceId`: assert `detached`, no grant, no `pty:exit -1`, and that provider state and ownership are NOT cleared. Mutation: restore the block at ssh-relay-session.ts:2641-2647. +7. Detach mid-publish does not release. Force a `session:set` flush between `setTabLayout(source)` and `setTabLayout(target)`: assert ownership is NOT released. Mutation: release on any publish missing the leaf. +8. Pane/tab delete releases. Delete a tab while its shell is live: assert the record is released, the shell IS an orphan, and no respawn is authorized. Mutation: remove the release rule — the shell must become invisible to both adopt and the cleanup UI. +9. Production id shapes in the recovery grant (E-1). Seed relay-native `pty-7`; register the runtime pty as `ssh:@@pty-7` with a NON-NULL connectionId; call `recoverTerminalPane`; assert the actual behavior. Mutation: swap either namespace. +10. E-2 false not-found. Same `relayInstanceId`, not-found caused by the wrong id form while the shell is in the Map under the correct id: assert NO `terminated` and NO grant. Mutation: skip `toRelaySshPtyId` on the attach path. +11. Projection is incarnation-scoped. Record `pty-1@incX`; restart the relay; it mints `pty-1@incY` for a different pane: assert the fresh shell IS an orphan and the stale record does NOT mask it. Mutation: subtract on ptyId alone. +12. `paneKey` survives `admit`. Mutation: remove it from the allowlist rebuild (pty-process-list-admission.ts:106-115). +13. Detached shell is still re-adopted. Seed a leafId-less lease whose relay-reported `paneKey` names a DIFFERENT tab than the live pane: assert the shell is attached exactly once and no second shell spawns. Mutation: match on full-string paneKey. This is the migration gate — green before K, re-run after. +14. Paired-client visibility survives the state rename. Record in `detached`, ptyId absent from every layout: assert `buildHeadlessMobileSessionTerminalTabs` still yields the tab. Mutation: drop the `expired`-equivalent — orca-runtime.ts:5964/:6290/:6322/:6359 must all be covered, not just the dead :16478 caller. +15. Cleanup UI reaches the user's own abandoned shells. Delete the repo row (new `repoId` on re-add) while shells run: assert they are offered for cleanup. Mutation: reinstate worktreeId-based "owned elsewhere" — must redden. +16. Nothing is killed implicitly. Any reattach/cleanup pass over foreign or unrecognized shells: assert zero kill RPCs. Mutation: auto-kill unclassified orphans. +17. One pane, one live shell across partitions. Two live shells for one `(worktree, tab, leaf)` with the binding only in `local`: assert exactly one is bound. Mutation: restore the ssh-first preference in `durablyBoundPtyIdForPane` (persistence.ts:7271). +18. P covers readers. Seed a binding only in `local`, call stable-pane resolve/retire with `connectionId` set: must find it. Mutation: revert either reader. +19. Adopt is exclusive. Two concurrent adopts of one orphan: one succeeds, the other gets a typed refusal. +20. Renderer cannot clobber ownership. Publish a stale full `WorkspaceSessionState` via `session:set`: assert the ownership map is byte-identical. +21. Offline quit path. Restorable predicate with store only and a null mux: must not throw, must not require an RPC (persistence.ts:6465). +22. Folder-workspace suffix. Two `::workspace:` instances on one path: a record for A must not reattach into B. +23. Target reassign carries ownership. `reassignSshTargetId(old, new)`: every record's `hostId` field and the reverse index follow, with no re-keying. +24. Windows/WSL relay kill. Assert the rules produce `detached`, never `terminated`, and that no UI copy asserts death. Mutation: assert-death — must redden on a Windows fixture even though it would pass on POSIX. +25. Old-relay degradation. `listProcesses` without `paneKey`, grant without `relayInstanceId`: projection non-empty, nothing killed, reattach falls back to the `getPtyIdsForConnection UNION records` work list, and the per-attach incarnation guard still rejects a recycled id. + +===================================================================== +D12. RE-TEST AGAINST THE SECTION-3 REFUTATIONS +===================================================================== +REFUTATION A LEG 1 — "a lease must name a live shell no pane owns." SURVIVES. Discovery never was the lease's job: reattach builds its list from `getPtyIdsForConnection(target) UNION` the durable rows (ssh-relay-session.ts:2211-2219), and the connect-time projection adds host-reported shells on top. Leaf-keying does not weaken this. D5's RELEASE-vs-RETIRE distinction is REQUIRED BY this leg: a record whose leaf is gone must be released (shell stays discoverable and adoptable) rather than deleted (knowledge lost) or terminated (respawn wrongly authorized). + +REFUTATION A LEG 2 — "a lease outlives the binding; expiry authorizes a 30s recreate." First clause survives exactly: `detached`/`terminated` are states on a main-owned, top-level record that persists after the layout binding is wiped. The recreate clause is PRESERVED AS A CONDITIONAL, not deleted: E-2 keeps a grant, gated on `relayInstanceId` equality, and only after E-1 shows the branch executes at production id shapes. Deleting the grant outright — proposed twice — is still refused as a simplification, because it dies to A leg 2 as written and rests on a read-not-run conclusion; it remains available as an E-1 OUTCOME. What does ship regardless is the user affordance, which covers every case a narrowed grant would drop. + +REFUTATION B — "make the host authoritative." SURVIVES, and is honored more cheaply than before. Ownership stays on local disk, top-level, main-owned, read synchronously inside `setLocalWorkspaceSession` during quit with the host down (D8). Everything this design changes is the record's KEY and WHAT MAY TRANSITION IT — never where it lives or how it is read. The host is consulted only while connected, and only for "which shells are in your Map?", "is this the same shell I bound?" (incarnation) and "are you the same process?" (relayInstanceId) — three questions it answers from data it already holds. Critically, D4's deletion of retire-on-absence is what keeps refutation B intact under load: a design that retired durable ownership because a remote listing timed out would have made the host authoritative by accident. + +REFUTATION C — a second authority architecture. Untouched, and this revision deletes more than the last: no decision table, no `hostEpoch`, no `os.uptime` math, no per-platform boot semantics, no paneKey transfer/alias layer, no attach-time identity apparatus, no orphan-ownership classifier, no second durable record. + +WHERE THE CUT IS. Plane 1 (delivery identity, credit ledger, chunk-and-ack, binary payloads, loopback WebSocket) proceeds in parallel. The cut is DURABILITY SCOPE vs EVIDENCE SOURCE, and this revision now applies that rule to all four inferences the prior round left standing: an error string means dead (deleted, D2), absence from a list means dead (deleted, D4/D5), a delivery-checkpoint flag means alive (deleted, D3), an arithmetic clock means rebooted (never built, D3). What remains on the wire is two optional fields: `relayInstanceId` on the grant and `paneKey` on the process summary. RC3 is entirely Plane 1's. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md new file mode 100644 index 00000000000..ca48bdb9e64 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md @@ -0,0 +1,255 @@ +# Terminal session correctness — design overview + +Plain-language explanation of what breaks, why, and what we changed. No prior +context needed. + +--- + +## 1. The cast of characters + +When you look at a terminal in Orca, five different things are lined up behind +that one rectangle on screen. + +```mermaid +flowchart LR + subgraph YOU["What you see"] + PANE["Pane
a rectangle in a tab"] + end + subgraph APP["Orca's bookkeeping"] + BIND["Binding
this pane uses that shell"] + LEASE["Lease
a note saying: we own
a remote shell"] + end + subgraph MACHINE["The actual computer"] + PTY["PTY
the shell process"] + HOST["Host
your Mac, or an SSH box"] + end + + PANE -->|bound to| BIND + BIND -->|points at| PTY + LEASE -->|claims| PTY + PTY -->|runs on| HOST +``` + +The pane is what you see. The **PTY** is the real shell process doing the work. +The **binding** is the note that connects them. On a remote machine there is also +a **lease** — a second note saying "we still own that remote shell." + +**Everything that went wrong is one of those notes pointing at the wrong thing.** + +--- + +## 2. The bug, in one picture + +You are working on an SSH machine. Your network hiccups. Orca reconnects. + +**What should happen:** your two terminals come back, same shells, same work. + +**What actually happened:** + +```mermaid +flowchart TD + A["You have 2 terminals"] --> B["Network hiccup"] + B --> C["Orca reconnects"] + C --> D["Now you have
19 terminals"] + D --> E["Hiccup again"] + E --> F["Now 20.
Most are ghosts you
never opened"] + + style D fill:#ffe6e6,stroke:#cc0000 + style F fill:#ffcccc,stroke:#cc0000 +``` + +That is a real report: **2 → 19 → 20**. Meanwhile the remote machine fills up +with shells nobody is using, until it refuses to start any more. + +A second, worse version: your AI coding agent gets **resumed twice**. Two copies +of `claude` writing into the same conversation file. One report had **five**. + +--- + +## 3. Why it happened — three root causes + +### Cause 1: the lease forgot which pane it belonged to + +Each lease is a note saying "we own remote shell X." The note _had_ a field for +"which pane," but nobody checked it. + +```mermaid +flowchart LR + subgraph BEFORE["Before — lease identified by shell only"] + P1["Pane A"] --> L1["Lease: shell-1"] + P1 -.->|reconnect makes
a new shell| L2["Lease: shell-2"] + P1 -.->|and another| L3["Lease: shell-3"] + N1["Nothing retires
the old notes"] + end + style N1 fill:#ffe6e6,stroke:#cc0000 +``` + +Every reconnect added a note and removed none. Next reconnect, Orca dutifully +restored **all** of them — one pane per stale note. + +**Fix:** a lease is now identified by _which pane_ it belongs to, not just which +shell. One pane, one live lease. Older notes are retired. + +> Retired means **marked expired, not killed**. The remote shell is deliberately +> left running — losing a note is not proof the shell died. + +### Cause 2: reconnecting could _create_ panes + +The function that says "pane A uses shell X" was also allowed to _invent_ pane A +if it didn't exist. That is correct when you open a new terminal. It is very +wrong when reconnecting. + +```mermaid +flowchart TD + R["Reconnect finds a
lease for shell-7"] --> Q{"Does its pane
still exist?"} + Q -->|"Yes"| OK["Reattach. Correct."] + Q -->|"No — before"| BAD["Invent the pane.
Ghost appears."] + Q -->|"No — after"| GOOD["Refuse. Leave the shell
running, unbound."] + + style BAD fill:#ffcccc,stroke:#cc0000 + style GOOD fill:#e6ffe6,stroke:#00aa00 +``` + +**Fix:** reattach now says _"bind only, never create."_ Spawning a new terminal +still creates, exactly as before. + +### Cause 3: a living shell was reported as dead + +The remote side has two different bad-news messages: + +| Real meaning | What it was reported as | +| ------------------------------------------------------------- | ------------------------------ | +| "This shell is gone." | `SESSION_EXPIRED` ✅ correct | +| "Shell is fine, I just need to re-establish the output pipe." | `SESSION_EXPIRED` ❌ **wrong** | + +Orca's reaction to "expired" is to start a fresh shell — and because it passes +the old session id along, the AI agent gets resumed a second time. **That is the +duplicate-agent bug.** + +**Fix:** the second case gets its own message, and respawning now requires +_positive proof_ the shell is gone. Anything unrecognised means "unknown", and +unknown never justifies starting a replacement. + +--- + +## 4. The one rule underneath all of it + +```mermaid +flowchart LR + E["Evidence"] --> A{"What does it
prove?"} + A -->|"Shell exited"| D["DEAD — cleanup allowed"] + A -->|"Can't reach it"| U["UNKNOWN"] + A -->|"Timed out"| U + A -->|"Not in a list"| U + A -->|"Tried 3 times"| U + U --> N["Do nothing destructive.
Wait. Retry. Ask."] + + style D fill:#e6ffe6,stroke:#00aa00 + style U fill:#fff4e6,stroke:#dd8800 + style N fill:#fff4e6,stroke:#dd8800 +``` + +**Unknown is not dead.** A disconnect, a timeout, an empty list, or three failed +retries tell you nothing about whether a process is alive. Every bug above is +some version of treating silence as a death certificate. + +We also made the liveness check able to _say_ "unknown". It used to return only +yes/no — so a provider whose network was down had no way to express "I can't +see, don't ask me" and answered "no", meaning dead. + +--- + +## 5. What we changed + +Six changes, ~214 lines of production code total. + +| # | Change | Stops | +| --- | ----------------------------------------------- | ----------------------------------------------- | +| 1 | Reattach binds, never creates | Ghost panes | +| 2 | Leases identified by pane | 2 → 19 → 20 growth | +| 3 | Heal already-corrupted lease data | Existing installs stuck at 20 | +| 4 | "Needs re-establishing" ≠ "expired" | Duplicate agent resume | +| 5 | Liveness can answer "unknown" | Live shells declared dead | +| 6 | One PTY's failure can't drop the shared channel | One bad pane killing every session on that host | + +### Change 6 in a picture + +```mermaid +flowchart TD + subgraph B["Before"] + X1["Pane 3 can't re-prove
its output stream"] --> X2["Give up after N tries"] + X2 --> X3["Drop the entire
connection to that host"] + X3 --> X4["Every pane, file transfer
and git command dies"] + end + subgraph A["After"] + Y1["Pane 3 can't re-prove
its output stream"] --> Y2["Park pane 3 only"] + Y2 --> Y3["Its shell keeps running.
Siblings untouched."] + end + style X4 fill:#ffcccc,stroke:#cc0000 + style Y3 fill:#e6ffe6,stroke:#00aa00 +``` + +### And a guard on every keystroke + +Orca queues your typing. If a pane switches to a different shell mid-flight, +queued keystrokes could land on the **new** shell. + +```mermaid +flowchart LR + K["Keystroke for shell-1"] --> C{"Is shell-1 still
this pane's shell?"} + C -->|"Yes"| S["Deliver"] + C -->|"No — pane moved on"| R["Refuse"] + C -->|"Pane unknown"| S2["Deliver
unknown ≠ stale"] + + style R fill:#fff4e6,stroke:#dd8800 +``` + +We did this **without changing the wire format** — Orca already keeps two lookup +tables in lock-step, so if they disagree, that _is_ proof the id is stale. No new +data is sent per keystroke. + +--- + +## 6. The approach we rejected + +A previous attempt built a **new authority architecture beside the old one**: ++60,903 lines of production code. It fixed **none** of the three root causes — +they were still present, untouched, underneath. + +```mermaid +flowchart LR + subgraph W["Rejected"] + O1["Old system
(3 bugs)"] + N1["New system
+60,903 lines"] + O1 -.->|"bugs still here"| O1 + end + subgraph R["Taken"] + O2["Old system
(3 bugs fixed)
+214 lines"] + end + style N1 fill:#ffcccc,stroke:#cc0000 + style O2 fill:#e6ffe6,stroke:#00aa00 +``` + +A second proposal of mine was also rejected in review — it would have added a +_second_ identity system beside an existing one, which is the same mistake in +miniature. Three of its factual claims turned out to be false when checked +against the code. + +**The lesson both times:** the machinery mostly existed already. The bug was that +it wasn't being _called_. + +--- + +## 7. What is still open + +| Thing | State | +| ------------------------ | -------------------------------------------------------------------------------- | +| The three root causes | Fixed | +| Duplicate agent resume | Root cause fixed; the codebase still carries 3 older patches for it | +| Orphaned remote shells | Left running on purpose. Not reachable from the UI yet — needs a recovery screen | +| The keystroke quarantine | Still needed. We proved deleting it re-opens a real `rm -rf` hazard | + +**On that last one:** there is a safety module that drops the tail of a half-typed +line after a recovery, because `echo hi; rm -rf x` can otherwise arrive as +`cho hi; rm -rf x` — the shell rejects `cho` and then happily runs `rm -rf x`. We +tried to delete it. We proved by experiment that we cannot, yet. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html new file mode 100644 index 00000000000..afe8bcbbc86 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html @@ -0,0 +1,275 @@ + + + + + + Terminal Architecture — Finalized Design + + + +
+
+

Terminal Architecture — Finalized Design

+

Synthesis of the “one protocol, three routes” proposal with a control-plane track it does not cover

+
+ +
+
Adopt
Data-plane proposal
+
+1 track
Control plane (missing)
+
REFUTED
“Delete the lease”
+
~374
Lines of reconciliation
+
~390
Remoteness branches
+
+ +
+

Verdict in one paragraph

+

The proposal is correct and its root cause is real — I verified it. Adopt it. But it is a data-plane design: framing, encoding, credit windows, backpressure. The bug that started this work is a control-plane bug, and none of its thirteen steps touch the records involved. The two tracks are independent and can run in parallel. My shipped work is a stopgap on the second track, framed around a symptom rather than a cause, and most of it should be deleted once the real fix lands.

+
+ +

1 · Two planes, one system

+

Nearly every disagreement here dissolves once you separate these.

+
+
What each plane owns⤢ zoom
+

proposal covers this

proposal is silent

CONTROL PLANE — who owns what

which pane owns
which shell

what survives
a restart

what may be
destroyed

DATA PLANE — moving bytes

framing · encoding

credit windows
backpressure

reconnect & resume

13 steps

0 steps
2 → 19 → 20 survives all 13

proposal covers this

proposal is silent

CONTROL PLANE — who owns what

which pane owns
which shell

what survives
a restart

what may be
destroyed

DATA PLANE — moving bytes

framing · encoding

credit windows
backpressure

reconnect & resume

13 steps

0 steps
2 → 19 → 20 survives all 13

+
+

Verified: none of the ladder’s core files reference the lease or binding records at all. You could ship all thirteen steps and still get 19 ghost panes on the third reconnect.

+ +

2 · The data-plane root cause — verified

+

This is the strongest finding in either document, and it is correct.

+

Orca copied a well-known persistent protocol’s 13-byte frame header, and kept two of its nine message types:

+ 
export const MessageType = { Regular: 1, KeepAlive: 9 } as const
+// dropped: Control · Ack · Disconnect · ReplayRequest · Pause · Resume · None
+

The seven dropped are exactly the ones that make the protocol persistent. The ACK field that survived feeds only unackedTimestamps — a seq→timestamp map read solely by the death timer. Nothing is retained; nothing is replayed.

+
+
Why three ledgers exist⤢ zoom
+

transport gives
no delivery guarantee

SSH stack rebuilds
retain + replay

runtime RPC rebuilds
retain + replay

daemon rebuilds it
lossily on purpose

three ledgers · three sets of bugs
4,847 lines in one of them

transport gives
no delivery guarantee

SSH stack rebuilds
retain + replay

runtime RPC rebuilds
retain + replay

daemon rebuilds it
lossily on purpose

three ledgers · three sets of bugs
4,847 lines in one of them

+
+
+

This reframes the duplication as three teams correctly solving a missing primitive, not carelessness. Fix the transport and the duplication loses its reason to exist. That is a genuine root cause — the framing I shipped earlier (“identity compared with the wrong key”) named a symptom class instead.

+
+ +

3 · Your principle, confirmed twice over

+

“SSH should behave like local, plus its own reconnection logic.” The codebase already proves it — for the other routes.

+ 
function getProvider(connectionId) {
+  if (!connectionId) return localProvider          // local AND daemon — one slot
+  const provider = sshProviders.get(connectionId)  // SSH — a parallel registry
+}
+
+
Three strategies, one exception⤢ zoom
+

the exception

SSH
neither

rides the local path
wearing a connectionId tag

~390 remoteness branches
+ 25 instanceof checks

works today

daemon
substitutes

swap the provider slot
0 branches added

paired runtime
encapsulates

hidden behind a transport
mostly negative guards

the exception

SSH
neither

rides the local path
wearing a connectionId tag

~390 remoteness branches
+ 25 instanceof checks

works today

daemon
substitutes

swap the provider slot
0 branches added

paired runtime
encapsulates

hidden behind a transport
mostly negative guards

+
+

The daemon is a 3,604-line provider that adds zero branches to the shared spine. SSH gets a second registry, and the tag then leaks outward as a condition ~390 call sites re-derive independently. Your instinct was not a preference — SSH is the only route that broke a pattern the codebase already implements correctly twice.

+ +

4 · What I got wrong

+ + + + + + + + +
ClaimStatusEvidence
“Identity compared with the wrong key” is the root causeSymptomThe real cause is one layer down — a transport with no delivery guarantee, and a tag instead of a substitution
My change consolidatesInverted~310 of ~405 added lines are SSH-only special case; every new function in the largest file is lease machinery
The superseded-keystroke fence protects reconnectInert thereIts maps have two writers, both spawn paths; relay reattach never populates them, so the guard permits every reattached PTY
“Less code proves a better design”Too bluntThe right control-plane fix is roughly net-neutral in lines; the payoff is a dissolved failure class, not a smaller diff
+
+

The fence failure is the thesis in miniature

+

A guard placed on the shared spine silently does not apply to the one route that does not ride the spine. That is not a coding slip — it is the predicted failure mode of tag-based divergence, and it happened to me while I was writing the fix for it.

+
+ +

5 · The control-plane defect

+

Two questions, one record, keyed for neither.

+
+
The key inversion⤢ zoom
+

today: one record

two different questions

which shell does
this pane own?

which shells exist
that nobody owns?

lease
keyed by (target, ptyId)
pane identity is optional payload

pane uniqueness is not structural
→ bolt on supersession, ranking,
rollback, dual matchers

~374 lines of reconciliation
9 states where records disagree

today: one record

two different questions

which shell does
this pane own?

which shells exist
that nobody owns?

lease
keyed by (target, ptyId)
pane identity is optional payload

pane uniqueness is not structural
→ bolt on supersession, ranking,
rollback, dual matchers

~374 lines of reconciliation
9 states where records disagree

+
+

Because the lease is keyed by shell rather than pane, “one pane, one live shell” is not expressible — it has to be enforced. My +209 lines are that enforcement: supersession, rank comparison, deferral, rollback. They exist only to compensate for the key.

+ +

Why the obvious fixes are both wrong

+ + + + + + +
ProposalVerdictWhy
Merge the lease into the pane bindingBlockedA lease must be able to name a live shell no pane owns, and must outlive the binding — expiry deletes the binding, then authorizes recovery for 30s
Delete the lease; let the host be authoritativeRefutedSee below — tested adversarially and it failed on all five probes
+
+
Why the host cannot be the authority⤢ zoom
+

no

no

no

make the remote host
the source of truth

does it hold
the state durably?

PTY table is an in-memory Map
no persistence layer at all

can the client
read pane identity back?

listProcesses and attach never return it
attach only compares, then throws
the one RPC that returns it has 0 callers

is the host reachable
when it matters?

the decisive one:
a synchronous disk read during quit
cannot become an RPC to a host that is down

no

no

no

make the remote host
the source of truth

does it hold
the state durably?

PTY table is an in-memory Map
no persistence layer at all

can the client
read pane identity back?

listProcesses and attach never return it
attach only compares, then throws
the one RPC that returns it has 0 callers

is the host reachable
when it matters?

the decisive one:
a synchronous disk read during quit
cannot become an RPC to a host that is down

+
+

I verified all three personally. The client-side record is load-bearing precisely because it works offline.

+ +

6 · The proposed control-plane fix

+

Don’t merge the records and don’t delete one. Give each question the key that answers it.

+
+
Split by question, not by transport⤢ zoom
+

after

OWNERSHIP
keyed by (host, pane)
same key local already uses

INVENTORY
keyed by (host, ptyId)
shells with no owner

which shell does
this pane own?

which shells exist
that nobody owns?

one pane, one shell
structural, not enforced

reattach work list ·
disconnected cleanup UI

supersession, ranking, rollback,
dual matchers all become unnecessary

after

OWNERSHIP
keyed by (host, pane)
same key local already uses

INVENTORY
keyed by (host, ptyId)
shells with no owner

which shell does
this pane own?

which shells exist
that nobody owns?

one pane, one shell
structural, not enforced

reattach work list ·
disconnected cleanup UI

supersession, ranking, rollback,
dual matchers all become unnecessary

+
+

Ownership becomes the same shape for local, daemon and SSH — one record per pane per host, with transport-specific payload (lease state and timestamps for SSH, nothing extra for local). That is your principle applied at the record layer. Inventory becomes its own small list, because “a live shell nobody owns” is genuinely a different fact, not a degenerate ownership row.

+

The 30-second recovery grant survives as a terminal state on the pane-keyed record, so nothing load-bearing is lost. And the offline consumers keep reading local disk synchronously, exactly as today.

+ +

7 · Where the proposal needs correcting

+ + + + + + + + +
Claim in the proposalAssessment
“Main is no longer in the data path at all” for a phone on an SSH hostOverstated — the pivot step forwards the host’s loopback socket to a port on your machine. Pairing, E2EE and the broker remain main’s. The ledger duplication collapses; the hop does not.
Target is “one wire contract”Incomplete — no step migrates the local daemon onto it. After the final step you have two protocols, not one. There is an unstated additional step, and it is large.
“The method layer is already transport-agnostic”True, and it volunteers its own counterexample. But 191 remoteness branches sit in the renderer above that boundary — including five separate re-declarations of one predicate — and no step addresses them.
Measured encoding cost (wire and CPU)Honestly reported — it states plainly that the absolute throughput is not a bottleneck, and argues from where the cost is paid instead. Not oversold.
+ +

8 · Combined roadmap

+
+
Two independent tracks⤢ zoom
+

independent — run in parallel

CONTROL PLANE — added here

pin behaviour with
oracles that bite

re-key ownership
by pane

split out
orphan inventory

delete the
compensating code

DATA PLANE — adopt as written

see it · capability hello

unify types · ledger ·
chunk-ack · lanes

binary payloads

boring first connect

SSH becomes
a port-forward

independent — run in parallel

CONTROL PLANE — added here

pin behaviour with
oracles that bite

re-key ownership
by pane

split out
orphan inventory

delete the
compensating code

DATA PLANE — adopt as written

see it · capability hello

unify types · ledger ·
chunk-ack · lanes

binary payloads

boring first connect

SSH becomes
a port-forward

+
+

The tracks touch disjoint files, so they can proceed simultaneously. The control-plane track should land its oracles first, because re-keying a durable record is a migration with a corruption story, and the existing tests pin the current key rather than the intended behaviour.

+ +

9 · What happens to the shipped work

+ + + + + + + + + + +
ChangeNowAfter the control-plane track
Reattach binds, never createsHolds the lineKeep — correct at any key
Pane-keyed lease supersession (+209)Holds the lineDelete — the key makes it structural
Corrupted-lease healingNeeded for existing installsFolds into the migration
“Needs re-establishing” ≠ “expired”Fixes duplicate agent resumeKeep — an error-taxonomy fix, orthogonal
Three-valued livenessCorrect generalizationKeep
Superseded-keystroke fenceInert on reattachFix or withdraw — decision needed
+ +

10 · Verification status

+

Stated explicitly, because two earlier claims in this program were retracted after checking.

+ + + + + + + + + + + + +
FindingStatus
Only 2 of 9 message types kept; ACK feeds the death timer onlyPersonally verified
Daemon substitutes via the provider slot; SSH uses a parallel registryPersonally verified
Pane maps have two writers, both spawn; fence inert on reattachPersonally verified
Relay PTY table is in-memory only; identity RPC has no production callerPersonally verified
Lease consulted from a synchronous, offline disk pathPersonally verified
Method layer transport-agnostic (0 / 0 / 27 references)Personally verified
~390 branch count, LOC ratios, the 9 disagreement states, ~374 reconciliation linesAgent-reported, spot-checked
Encoding throughput measurements in the proposalNot re-measured
+ +

11 · Decisions needed

+ + + + + + + + +
#Decision
1Adopt the data-plane proposal as written? My review found no defect in its reasoning — only two overstated claims and one missing step. It is independent of everything below.
2Approve the control-plane track? Re-keying a durable record needs a migration and a corruption story. It is roughly net-neutral in lines and deletes a failure class rather than a file.
3The inert fence — populate the pane maps on the reattach path, or withdraw the guard and its test? Shipping it as-is means a test asserting a protection that does not apply where it matters.
4The open PRs — merge as a stopgap that keeps the reported bug closed, or hold them and go straight at the control-plane track?
+ +
+ Companion to report.html (evidence for the shipped work). Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
×
scroll to zoom · drag to pan · Esc to close
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql b/docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql new file mode 100644 index 00000000000..5008d103cec --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql @@ -0,0 +1,324 @@ +query TerminalSessionCorrectnessRelatedWork { + repository(owner: "stablyai", name: "orca") { + n8129: issueOrPullRequest(number: 8129) { + ...IssueDetails + ...PullRequestDetails + } + n8275: issueOrPullRequest(number: 8275) { + ...IssueDetails + ...PullRequestDetails + } + n8585: issueOrPullRequest(number: 8585) { + ...IssueDetails + ...PullRequestDetails + } + n8618: issueOrPullRequest(number: 8618) { + ...IssueDetails + ...PullRequestDetails + } + n9034: issueOrPullRequest(number: 9034) { + ...IssueDetails + ...PullRequestDetails + } + n9092: issueOrPullRequest(number: 9092) { + ...IssueDetails + ...PullRequestDetails + } + n9093: issueOrPullRequest(number: 9093) { + ...IssueDetails + ...PullRequestDetails + } + n9138: issueOrPullRequest(number: 9138) { + ...IssueDetails + ...PullRequestDetails + } + n9562: issueOrPullRequest(number: 9562) { + ...IssueDetails + ...PullRequestDetails + } + n9585: issueOrPullRequest(number: 9585) { + ...IssueDetails + ...PullRequestDetails + } + n9819: issueOrPullRequest(number: 9819) { + ...IssueDetails + ...PullRequestDetails + } + n9820: issueOrPullRequest(number: 9820) { + ...IssueDetails + ...PullRequestDetails + } + n9827: issueOrPullRequest(number: 9827) { + ...IssueDetails + ...PullRequestDetails + } + n9833: issueOrPullRequest(number: 9833) { + ...IssueDetails + ...PullRequestDetails + } + n10118: issueOrPullRequest(number: 10118) { + ...IssueDetails + ...PullRequestDetails + } + n10208: issueOrPullRequest(number: 10208) { + ...IssueDetails + ...PullRequestDetails + } + n10235: issueOrPullRequest(number: 10235) { + ...IssueDetails + ...PullRequestDetails + } + n10385: issueOrPullRequest(number: 10385) { + ...IssueDetails + ...PullRequestDetails + } + n10415: issueOrPullRequest(number: 10415) { + ...IssueDetails + ...PullRequestDetails + } + n10744: issueOrPullRequest(number: 10744) { + ...IssueDetails + ...PullRequestDetails + } + n11006: issueOrPullRequest(number: 11006) { + ...IssueDetails + ...PullRequestDetails + } + n11265: issueOrPullRequest(number: 11265) { + ...IssueDetails + ...PullRequestDetails + } + n11339: issueOrPullRequest(number: 11339) { + ...IssueDetails + ...PullRequestDetails + } + n11342: issueOrPullRequest(number: 11342) { + ...IssueDetails + ...PullRequestDetails + } + n11495: issueOrPullRequest(number: 11495) { + ...IssueDetails + ...PullRequestDetails + } + n11574: issueOrPullRequest(number: 11574) { + ...IssueDetails + ...PullRequestDetails + } + n11575: issueOrPullRequest(number: 11575) { + ...IssueDetails + ...PullRequestDetails + } + n11596: issueOrPullRequest(number: 11596) { + ...IssueDetails + ...PullRequestDetails + } + n11622: issueOrPullRequest(number: 11622) { + ...IssueDetails + ...PullRequestDetails + } + n11729: issueOrPullRequest(number: 11729) { + ...IssueDetails + ...PullRequestDetails + } + n11800: issueOrPullRequest(number: 11800) { + ...IssueDetails + ...PullRequestDetails + } + n11801: issueOrPullRequest(number: 11801) { + ...IssueDetails + ...PullRequestDetails + } + n11803: issueOrPullRequest(number: 11803) { + ...IssueDetails + ...PullRequestDetails + } + n11804: issueOrPullRequest(number: 11804) { + ...IssueDetails + ...PullRequestDetails + } + n11904: issueOrPullRequest(number: 11904) { + ...IssueDetails + ...PullRequestDetails + } + n12036: issueOrPullRequest(number: 12036) { + ...IssueDetails + ...PullRequestDetails + } + n12134: issueOrPullRequest(number: 12134) { + ...IssueDetails + ...PullRequestDetails + } + n12140: issueOrPullRequest(number: 12140) { + ...IssueDetails + ...PullRequestDetails + } + n12220: issueOrPullRequest(number: 12220) { + ...IssueDetails + ...PullRequestDetails + } + n12241: issueOrPullRequest(number: 12241) { + ...IssueDetails + ...PullRequestDetails + } + n12264: issueOrPullRequest(number: 12264) { + ...IssueDetails + ...PullRequestDetails + } + n12339: issueOrPullRequest(number: 12339) { + ...IssueDetails + ...PullRequestDetails + } + n12447: issueOrPullRequest(number: 12447) { + ...IssueDetails + ...PullRequestDetails + } + n12448: issueOrPullRequest(number: 12448) { + ...IssueDetails + ...PullRequestDetails + } + n12456: issueOrPullRequest(number: 12456) { + ...IssueDetails + ...PullRequestDetails + } + n12474: issueOrPullRequest(number: 12474) { + ...IssueDetails + ...PullRequestDetails + } + n12477: issueOrPullRequest(number: 12477) { + ...IssueDetails + ...PullRequestDetails + } + n12568: issueOrPullRequest(number: 12568) { + ...IssueDetails + ...PullRequestDetails + } + n12600: issueOrPullRequest(number: 12600) { + ...IssueDetails + ...PullRequestDetails + } + n12634: issueOrPullRequest(number: 12634) { + ...IssueDetails + ...PullRequestDetails + } + n12683: issueOrPullRequest(number: 12683) { + ...IssueDetails + ...PullRequestDetails + } + n12699: issueOrPullRequest(number: 12699) { + ...IssueDetails + ...PullRequestDetails + } + n12702: issueOrPullRequest(number: 12702) { + ...IssueDetails + ...PullRequestDetails + } + n12743: issueOrPullRequest(number: 12743) { + ...IssueDetails + ...PullRequestDetails + } + n12749: issueOrPullRequest(number: 12749) { + ...IssueDetails + ...PullRequestDetails + } + n12751: issueOrPullRequest(number: 12751) { + ...IssueDetails + ...PullRequestDetails + } + n12760: issueOrPullRequest(number: 12760) { + ...IssueDetails + ...PullRequestDetails + } + n12768: issueOrPullRequest(number: 12768) { + ...IssueDetails + ...PullRequestDetails + } + n12798: issueOrPullRequest(number: 12798) { + ...IssueDetails + ...PullRequestDetails + } + n12882: issueOrPullRequest(number: 12882) { + ...IssueDetails + ...PullRequestDetails + } + n12903: issueOrPullRequest(number: 12903) { + ...IssueDetails + ...PullRequestDetails + } + n12987: issueOrPullRequest(number: 12987) { + ...IssueDetails + ...PullRequestDetails + } + n13013: issueOrPullRequest(number: 13013) { + ...IssueDetails + ...PullRequestDetails + } + n13110: issueOrPullRequest(number: 13110) { + ...IssueDetails + ...PullRequestDetails + } + n13111: issueOrPullRequest(number: 13111) { + ...IssueDetails + ...PullRequestDetails + } + p13110: pullRequest(number: 13110) { + ...ReviewDetails + } + p13111: pullRequest(number: 13111) { + ...ReviewDetails + } + } +} + +fragment IssueDetails on Issue { + number + title + url + state + stateReason + updatedAt +} + +fragment PullRequestDetails on PullRequest { + number + title + url + state + isDraft + merged + reviewDecision + mergeable + mergeStateStatus + headRefName + baseRefName + additions + deletions + changedFiles + updatedAt + commits(last: 1) { + nodes { + commit { + oid + statusCheckRollup { + state + } + } + } + } +} + +fragment ReviewDetails on PullRequest { + reviewThreads(first: 100) { + nodes { + isResolved + isOutdated + comments(last: 1) { + nodes { + url + author { + login + } + } + } + } + } +} diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md new file mode 100644 index 00000000000..d6842d0d563 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md @@ -0,0 +1,683 @@ +# Goalposts and proof contract + +This file answers one question: **what exact evidence is required before an +agent may write “proven”?** + +The current status is **0/8 proven goalposts** and **2/13 proven +journeys**. Status may change only from evidence produced on the same rebased, +converged release candidate. Historical snapshot tests and independently useful +narrow PRs may be cited as partial evidence but cannot promote a row. + +## Recorded user decisions + +These amend the contract above. Only the user may add to this list. + +### D1 — G6 relaxed from strictly net-negative (2026-08-07) + +> "It's okay if we must increase LOC but try not to." + +G6's pass condition is no longer a strict inequality against the frozen +baseline. It is now: **minimise added production code, and justify every net +addition against the correctness it buys.** A net-positive total does not fail +G6 on its own; an unjustified one does. + +Why this was needed: the deletion budget the plan assumed does not exist. An +entrypoint-rooted import graph over all 20 real build entrypoints found that +51 of the 53 candidate files in `src/relay/*pty-source-*`, +`src/shared/pty-consumer-*`, and `src/main/ipc/ssh-pty-*` are reachable and +value-instantiated on live paths. Only 2 files (263 LOC) are unreachable, and 2 +of the candidates did not exist at the baseline so deleting them earns no +credit. Against roughly +1,021 LOC to offset, a strict inequality was not +reachable without deleting load-bearing code — which the governing rule forbids. + +Still binding: correctness may not be weakened to reduce line count, and a +replacement architecture added beside the old one does not earn its lines. + +## Proven journeys + +### Journey evidence not yet sufficient to promote + +**Journey 2 (daemon half).** `tests/e2e/daemon-restart-session-liveness.spec.ts`, +3 tests. Two mutations each redden exactly one clause, on macOS and Linux: +reverting three-valued `hasPty` reddens only the unknown-not-dead clause; +widening the sole-provider fallback reddens only the stale-generation clause. +The lead re-verified the first mutation independently. **Missing: physical WSL**, +which the journey names explicitly. That host is now unblocked (the distro has a +provisioned default user) and the run is outstanding. + +**Journey 12.** `tests/e2e/cross-version-wire/`, 13 tests against a real published +baseline. Reverting the restore-required publication to expiry reddens 7 of 13; +restore greens. Lead-verified. **Missing: live skew.** These are in-process wire +tests. The original ledger named live paired-runtime and SSH skew as the gap, and +an in-process decoder test does not close it. + +**Journey 4.** Sibling-pane isolation on one host reddens when +`mux.dispose('connection_lost')` is restored. **The cross-host clause cannot be +proven by mutation**: a mux belongs to one relay session per SSH target, so its +dispose cannot cross a host boundary — the cross-host test stayed green under the +mutation, which is the empirical receipt. That clause rests on +isolation-by-construction, and saying otherwise would be a false claim. + +**Journey 6.** `tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts`, +4 tests against a real Docker OpenSSH container configured `MaxSessions 1`, the +cap read back from `sshd -T` inside each test. Three of the four clauses +discriminate, clause-selectively, in single runs (the spec is not `serial`, so a +red test never reports the others as "did not run"): + +- dropping `mayCreate: false` from the reattach binding write reddens only the + authority-reconnect test, with the unowned leaf grafted into the `local` + partition; +- marking leases `terminated` instead of `detached` in `beginShutdownDetach` + reddens only the two restart tests, with a cold-spawned shell beside the + surviving one. + +**Missing: the transport-disconnect clause does not discriminate.** Four guard +removals were tried and it stayed green under all of them, including publishing +`SSH_SESSION_EXPIRED` in place of `SSH_SOURCE_RESTORE_REQUIRED` and making +`isProvenSshSessionGoneError` return true for every error. A sever that leaves +the detached relay holding its delivery record reattaches through the checkpoint +path, so no reattach failure is ever classified and the respawn-requires-proof +guards are off that path. Inducing `restoreRequired` needs a lost or mismatched +delivery record, which this fault does not produce. That clause is therefore a +forward guard and cannot promote the row. Also missing: only Docker-on-macOS was +run; the explicit-close/proven-teardown retirement clause is not covered. + +**Journey 13.** Not proven. One of ten named dimensions measured. The +`isSupersededPtyId` fence costs roughly 14ns per call, but it was measured on +lifted predicates in plain Node, not through real Electron IPC — that part is an +inference, not a measurement. + +### Correction: Journey 12's cross-version tests exercise a route production does not have + +Recorded earlier that the cross-version suite "confirms the new +`SSH_SOURCE_RESTORE_REQUIRED` token mutates nothing on an old client." That +claim is withdrawn. + +An attempt to build a LIVE two-process skew oracle established that the token +never crosses a version boundary at all. It is minted in main +(`src/main/providers/ssh-pty-provider.ts:110`) and consumed in the same app's +renderer (`pty-connection.ts:8833`, `:9080`) over Electron IPC — always one +version. The only cross-version terminal boundary, paired client to HUB runtime, +goes through `remote-runtime-pty-transport.ts`, whose sole respawn trigger is +`SSH_SESSION_EXPIRED` (line 1481), and whose reattach never reaches +`SshPtyProvider.spawn`. + +The in-process suite injects the token into `terminal.resolvePane`, which is a +pure lookup and not a route production takes. So those 13 tests are a decoder +guard, not evidence about old clients. + +Two consequences, both good: the wire-compatibility risk this branch was thought +to introduce **does not exist**, because there is no version boundary for the new +token to cross. And Journey 12 needs a different oracle entirely — one aimed at a +boundary that is genuinely cross-version. + +### Journey 6 — evidence, and why it is not promotable + +`tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts`, 4 tests, each +with its own container and Electron profile, against real OpenSSH with +`MaxSessions 1` read back from `sshd -T` rather than assumed. Remote pids are +read on the container two independent ways and must agree, each carrying its +kernel start time. + +Two mutations discriminate and are disjoint: removing `mayCreate: false` reddens +only the reconnect clause; marking leases `terminated` instead of `detached` at +quit reddens only the two restart clauses. + +**Not promotable:** the "same remote PID and exact binding survive a transport +disconnect" clause is a forward guard. Four separate guard removals left it +green. It asserts a real property, but nothing shipped is load-bearing for it, +so it cannot be claimed as proof of this branch's work. + +### Journey 3 — evidence, and why it is not promotable + +`tests/e2e/ssh-lazy-discovery-skipped-host-restart.spec.ts`, 3 tests, two real +containers, sampling sshd's own accept log and live session census 20+ times over +22 seconds with the in-use host as a positive control. + +**Not promotable:** no mutation reddens clause 3 alone. The genuine cross-host +lease-scoping guard is load-bearing, but removing it breaks the sibling host +during setup, so the failure carries no clause information. + +### Journey 2 — Daemon and physical WSL (proven 2026-08-08) + +Oracle: `tests/e2e/daemon-restart-session-liveness.spec.ts`, 3 tests, plus +`tests/e2e/helpers/daemon-shell-process-identity.ts`. The PTY leader is a real +login shell that reports `$$` back through the production write path +(`DaemonPtyRouter.write` -> daemon socket -> PTY), and that pid is resolved to a +kernel start time, so a look-alike respawn cannot pass as a survivor. + +It replaced a spec that never crossed the daemon boundary and whose successor +generation owned nothing, which made "the live successor is neither killed nor +replaced" vacuous. + +| Environment | Result | Discrimination watched | +| -------------------------------------------- | -------- | --------------------------------------------------- | +| macOS 26.3.1 arm64 | 3 passed | 2 mutations, each reddening one clause | +| Ubuntu 24.04 x86_64 native | 3 passed | same 2 mutations, same single-clause reds | +| WSL2 Ubuntu 26.04 on a physical Windows host | 3 passed | reverting three-valued `hasPty` reddens test 1 only | + +Clause-selective on all three. The spec runs `mode: 'serial'`, so a red test 1 +reports 2 and 3 as "did not run" rather than passing — selectivity was therefore +established by re-running 2 and 3 _alone under the same mutation_ and watching +them stay green, not by assuming it. + +Mutation A (`hasPty` reverted to `activeSessionIds.has(id)`) reddens only the +unknown-not-dead clause. Mutation B (widening the sole-provider fallback in +`daemon-session-owner-resolution.ts`) reddens only the stale-generation clause. +The lead independently reproduced Mutation A on macOS. + +Also confirmed on the WSL host, independently of the spec: an Orca WSL-mode +terminal now starts and returns real output. It could not start before — the +distro had no provisioned default Unix user, so every interactive `wsl.exe` +launch blocked on first-run provisioning. + +Limits, stated rather than implied: + +- The WSL run used `ELECTRON_RUN_AS_NODE=1`, which is what this oracle needs + (real daemon processes and real PTYs, no renderer). It does **not** show that + Electron's GUI starts under WSL; no display server was started and that + remains an open question for any journey needing a window. +- WSL evidence is against `d74f5ed0eae` and on Node 22 rather than the required 24. The other two platforms ran the same oracle on Node 24. +- `src/main/providers` is red on that distro for an unrelated reason: + `local-pty-shell-ready.test.ts` pins an exact OSC 133 marker count and saw 7 + splits where it expects 4. The WSL run attributed that to bash 5.3.9, but that + is **not** the cause — macOS runs the same bash 5.3.9 and the spec passes there + 67/67. The trigger is environmental to that distro, most plausibly a + system-wide bashrc contributing prompt hooks, and the underlying defect is that + the spec asserts an absolute count of markers it does not own. Outside this + journey's surface, so left for its owner rather than reinterpreted blind — but + it means "the unit suites are green on WSL" would be false. + +### Journey 1 — Local macOS, Linux, and Windows (proven 2026-08-08) + +Oracle: `tests/e2e/local-terminal-restart-binding-identity.spec.ts`. Two tests — +the same pane, full binding and OS shell process survive renderer reload and app +restart; and a stale pre-spawn session write is rejected instead of retiring the +live binding. + +Process identity is proved by the shell reporting its own pid through the +production write path, plus the kernel-reported start time, so a recycled pid +cannot pass as a survivor. + +| Platform | Result | Discrimination watched | +| --------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| macOS 26.3.1 arm64 | 2 passed | mutation reddens; restore greens | +| Ubuntu 24.04 x86_64, native | 2 passed | 4 cycles, incl. deleting `restorableBindings` reconciliation — reddens test 2 only | +| Windows 11 26200, native | 2 passed | A: forcing `shutdownDaemon()` in will-quit reddens test 1 (pid 13756 -> 8852, start times 5.4s apart). B: disabling the same reconciliation reddens test 2 only | + +Every run used an isolated TMPDIR, because the harness keys its seeded-repo +pointer on a machine-global tmpdir path and concurrent runs can both fabricate +and mask a red. + +Selectivity holds on two platforms: mutation B reddens only the stale-operation +test and leaves the restart test green, so the two clauses are independently +proved rather than jointly. + +Residual limit, stated: "every stale exact operation" is proved for the +pre-spawn session-write class. Local `pty:write`/`resize`/`signal` are fenced by +`isSupersededPtyId` with its own oracle, but that fence compares a binding, not +an incarnation — see `binding-identity-design.md`. + +## Universal proof rule + +A gate is `proven` only when both exist: + +1. the complete behavior is reachable through every named production path; and +2. the named proof exercises that behavior and its failure boundaries on the + final candidate. + +Every proof receipt must record: + +- candidate commit SHA and exact merge base; +- unfixed baseline SHA used for red/green discrimination; +- exact command or manual journey protocol; +- OS, architecture, filesystem, runtime, Git, SSH, daemon, relay, client, and + host versions relevant to that journey; +- UTC start/end time, exit status, counts, and artifact/log location; +- the production caller, persistence boundary, transport, and cleanup path + exercised; +- the expected oracle and the observed result; and +- independent reviewer identity and unresolved-finding count. + +For a new regression oracle, demonstrate that it fails for the intended reason +on the unfixed baseline and passes on the candidate. A test that passes both is +a forward guard and cannot prove the fix. + +## G0 — One design contract + +**Current status: partial.** + +G0 is proven only when: + +- every reachable production path follows one reconciled identity, host + boundary, operations, delivery, migration, compatibility, and minimal-shape + contract; +- there is one final-host authority service, thin transport adapters, one app + projection/controller, and one bounded pre-cutover legacy importer; +- no adapter or renderer owns a parallel authority state machine; +- all retained existing primitives are mapped by semantics, not similar names; +- the lost-consumer/compaction liveness gap has a safe, bounded, authenticated, + operator-reachable resolution that does not infer death from time; and +- an independent production-graph audit finds no contradictory path. + +The design document alone, a behavior contract, or focused tests do not prove +G0. + +## G1 — One final-host authority + +**Current status: partial.** + +G1 is proven only when local, daemon, WSL, direct SSH, nested SSH, paired +runtime, and remote server all resolve: + +- a stable identity minted or validated by the final PTY-owning host; +- a canonical host-local namespace for worktree, folder, and floating + workspaces; +- one exact pane-generation/PTY-incarnation binding; +- host connections keyed by final-host identity with lazy discovery; +- concurrent host isolation; and +- namespace-local admission, failure, grant, handover, and retirement. + +`connectionId`, SSH target ID, client repository ID, path spelling, or +`worktreeId` may be routing metadata. Their existing names and usage counts are +not proof that they satisfy final-host identity or namespace semantics. + +Proof requires the applicable live journeys below, including simultaneous hosts +and independent client/host updates. + +## G2 — Exact operations only + +**Current status: partial.** + +G2 is proven only when input, resize, signal, close, output, and exit are fenced +by the full binding captured before any await: + +- authority host; +- namespace; +- pane generation; +- owner/writer incarnation; +- physical PTY; +- PTY incarnation; and +- negotiated operation/source generation where applicable. + +Stale, partial, absent, timed-out, disconnected, or unknown evidence must not +affect a successor. An authoritative operation must never retry through an +ID-only provider call or a legacy mutation path. + +Proof must cover all operations across local, daemon, WSL, SSH, paired runtime, +remote server, renderer fallback, restart, concurrent replacement, and both +mixed-version directions. Store-row counts and source-text assertions are not +sufficient. + +## G3 — Durable ordered delivery + +**Current status: partial.** + +G3 is proven only when the final production design provides all observable +properties below, even if its internal mechanism differs from the preserved +construction design: + +- complete boundary snapshot before later events; +- producer held while boundary/replay is established; +- contiguous replay before reconciliation and live resume; +- durable semantic outcomes, including exit and state needed by a newly + attached consumer; +- durable idempotent main-process projection before acknowledgement; +- final-host-owned cumulative acknowledgement or an explicitly approved + equivalent with the same crash/replay guarantees; +- renderer snapshot-plus-delta observation; +- app, renderer, host, and transport restart resume; +- gap detection and resnapshot without silent omission; +- bounded memory, queues, pages, listeners, timers, and retained output; +- independent consumer retirement and safe compaction liveness; and +- no app-side duplicate cursor, settlement, receipt, or suffix-reconciliation + authority. + +Proof must include crash-before/after-ACK cuts, lost responses, disconnected +replay, gap recovery, slow/stalled consumers, retired and permanently lost +consumers, paired/remote restart, mixed versions, and scale. + +## G4 — One-way legacy cutover + +**Current status: partial.** + +G4 is proven only when each namespace performs, in order: + +1. explicit capability negotiation; +2. a brief legacy-write freeze; +3. exact non-mutating inventory; +4. a deterministic import plan; +5. validation with ambiguity kept visible and non-destructive; +6. one self-contained durable authority commit; +7. topology attachment; and +8. exact client opening through the authoritative path. + +There must be no dual writer, destructive inference, authority-to-legacy +fallback, or second durable migration catalog after cutover. Old peers remain +on an unchanged isolated legacy surface or fail before mutation. + +Proof requires crash cuts at every phase, replay from the self-contained commit, +ambiguous-row isolation, independent namespace failure, and legacy-writer and +reconciliation deletion. + +## G5 — Wire and platform compatibility + +**Current status: not started.** + +G5 is proven only when all exchanged changes follow the remote-wire +compatibility contract and the final candidate passes: + +- old client to new host; +- new client to old host; +- native macOS; +- native Linux at Ubuntu 20.04 / glibc 2.31 compatibility floor; +- native Windows; +- physical WSL, including Git Bash/`.cmd` boundaries where relevant; +- Docker OpenSSH; +- daemon; +- direct and nested SSH; +- paired runtime; +- remote server; +- git worktree; +- folder workspace; +- floating workspace; and +- drive-letter and UNC namespace paths. + +Run both skew directions independently across every changed deployment boundary: +app↔daemon, app↔SSH relay/final host, paired client↔paired runtime, remote +client↔remote server, and mobile/E2EE RPC where affected. A single in-process +codec test or one client/host pairing cannot stand in for this matrix. + +New opcodes or semantics require explicit capability negotiation. An optional +field that parses on an old peer does not by itself prove that old behavior +remains correct. + +Mocking `process.platform`, running Linux inside Docker, or passing in-process +wire unit tests does not prove the corresponding native or live-skew row. + +## G6 — Simpler, minimised production code + +**Current status: partial; structural clauses remain open.** + +Decision D1 replaces the original strict inequality: the final integrated +program must minimise production source and justify every net addition against +the correctness it buys. + +The default baseline is +`5ed45739e94bdf6460364e033bfcec9b32c0b42a`, the base recorded by GitHub for +PR #12600. This broader program subsumes #12600. Changing the baseline requires +an explicit user decision recorded before more implementation begins. + +G6 is proven only when: + +- aggregate program-attributable production source net LOC is **minimised and + every net addition justified** against the frozen baseline — see decision D1 + under "Recorded user decisions", which replaced the original strict + inequality once the assumed deletion budget was shown not to exist; +- every program-attributable prerequisite merged after the baseline and every + stacked PR is included, even if a later rebase places it in `main`; +- overlapping changes are recomputed from the frozen baseline to the final tree + so additions and later deletions are not double-counted; +- unrelated upstream or user deletions cannot offset program additions; +- production, test, documentation, CI/runner, generated, and vendored changes + are reported separately; +- every new production module is reachable from a real entrypoint; +- no test fixture remains under production compilation; +- there is one identity comparison, transition implementation, exact-operation + client, mutation admission path, and delivery state machine; +- no re-export shim or one-type module exists solely to preserve construction + layering; +- no superseded quarantine, sliding-window, retry-verdict, reconciliation, + duplicate cursor, receipt ledger, legacy writer, or migration bridge remains + reachable after cutover; and +- an independent reachability and duplicate-state-machine audit is clean. + +Tests or docs cannot offset positive production LOC. A smaller narrow PR is not +proof if the final stack remains net positive. Deleting correctness or platform +coverage to hit the number is prohibited. + +Classify by behavior, not directory name: shipped runtime code, migration code, +and build-time code that enforces a shipped artifact invariant are production; +test-only runners and fixtures are tests even when misplaced, and their presence +under production compilation independently fails this gate. Publish the final +file-by-file classification so the count cannot be moved between buckets. + +### G6 clause assessment (2026-08-08) + +Checked against the current branch rather than assumed: + +| Clause | State | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| aggregate net production LOC minimised and justified (D1) | +228, each addition justified in its commit; 201 LOC deleted after verifying unreachability from all 20 build entrypoints | +| every new production module reachable from a real entrypoint | holds — no new unreachable module was added | +| no test fixture under production compilation | not audited on this branch | +| one identity comparison / transition / exact-op client / admission path / delivery state machine | **fails** — several remain, and this is the clause the rejected binding proposal would have made worse by adding a second identity comparison beside `resolveStablePaneOwner` | +| no re-export shim or one-type module for layering | holds for this branch's additions | +| **no superseded quarantine, sliding-window, retry-verdict, reconciliation, duplicate cursor, receipt ledger, legacy writer, or migration bridge reachable** | **fails** — `terminal-input-quarantine.ts` is imported by `pty-connection.ts:98` and `terminal-pane-recovery.ts:6` | +| independent reachability and duplicate-state-machine audit clean | not run | + +On the quarantine specifically: it is **not** subsumed by the superseded-PTY +fence, and claiming otherwise would be wrong. The fence refuses writes aimed at +a stale ptyId. The quarantine guards the user's _subsequent_ typing — the tail +of a half-typed line — landing on the successor shell under its current and +correct ptyId, which the fence never sees. + +Deleting it therefore requires contract property B: a _different_ shell surfaces +unresolved and never silently receives the pane, at which point there is no +mangled tail to suppress. That is a behaviour change to the recovery path, not a +deletion, and an earlier attempt to shortcut it was rejected for reintroducing +the `echo hi; rm -rf x` hazard by disarming per-pane state that is keyed per-tab. + +G6 is therefore not promotable, and its blocking clauses are structural rather +than arithmetic. + +### The input quarantine is load-bearing, not superseded (2026-08-08) + +G6's clause lists "no superseded quarantine ... remains reachable" and +`terminal-input-quarantine.ts` was assumed to be one. It is not, and the +evidence is direct. + +**The hazard is live.** Disabling the single call site in `pty-connection.ts` +and running the pane's own oracle reproduces it verbatim, lead-verified: + + AssertionError: expected "vi.fn()" to not be called with arguments: + [ 'cho hi; rm -rf x' ] + +Deleting the module without a replacement re-opens `rm -rf` execution. + +**Contract property B was costed by building it, not estimated.** Threading the +incarnation to the renderer compiles at **+26 production LOC**, and completing it +(publishing an incarnation from `resolveTerminalPane`, plus a new exported reader +for the module-private `ptyIncarnationById` map) is about **+33**. The +cross-remount state the comparison needs must outlive the destroyed pane, so it +becomes a module of roughly the size of the one being deleted — the two in-tree +precedents are 88 and 86 lines. Adding a renderer-side identity comparison would +also worsen G6's already-failing "one identity comparison" clause. Floor: about +**+140 production LOC to delete 88**. + +**And the route is not uniformly available.** `RuntimeTerminalCreate` and +`RuntimeTerminalResolvePane` carry no `incarnationId`; remote connect results are +built without one; and the transport latches `resolvePaneUnavailable` when a host +answers `method_not_found`. Mixed client/host versions are the normal state, and +an optional field does not make an old host publish it. So a paired client reads +_unknown_ — which under this program's own governing rule is not proof of "same +shell". Requiring proof makes every remote reattach surface unresolved; not +requiring it needs a fallback, and the only correct fallback is this quarantine. +Either way the module stays reachable. + +`endpointReplaced` is also routine rather than rare — a daemon death remounts +every live pane — so property B would convert each into a manual per-pane +reconnect, which is the coverage deletion G6 explicitly prohibits. + +**Consequence:** this clause of G6 cannot be closed by deletion. Either the +clause is amended to recognise the module as load-bearing, or the program accepts +a net-positive change to replace it with something weaker on remote hosts. That +is a user decision (call it D5), not one to assume. + +### G6 clause: test fixtures under production compilation (audited 2026-08-08) + +Audited by importer rather than filename, then checked against what the build +actually emits. **The clause is already satisfied on the meaning that matters, +and cannot be closed by moving files on the other meaning.** + +36 candidates; 4 have genuine production importers and are correctly placed. The +other 32 (~3,300 LOC) have zero non-test importers. + +**They do not ship.** Spot-checked the emitted bundle for +`terminal-restore-parity`, `ipcEventsTestHarness`, `storeTestHelpers` and +`sshRelayNativeDepsInstallFixture`: none appears in `out/`. Rollup drops them +because no production entrypoint reaches them. Under "compiles into the shipped +product", this clause holds today. + +**Moving them would close nothing under the other reading.** +`config/tsconfig.node.json` and `config/tsconfig.tc.web.json` declare `include` +globs such as `../src/main/**/*` with **no `exclude` at all**. A `__tests__/` +directory is matched by that glob exactly as any other path is, and so is every +`*.test.ts` file in the repo. Relocating 32 fixtures would not remove one file +from typecheck scope. + +A sweep was started and stopped once this was verified, rather than landing 32 +moves across areas this program does not own for no measurable gain. + +If the intent behind the clause is that typecheck scope should exclude test code, +that is a repo-wide tsconfig change affecting every test file — different work +with a different owner, and it should be stated as such rather than pursued by +relocating fixtures. + +One genuine defect was found and fixed while auditing: +`terminal-pane/xterm-bypass-event-fixture.ts` and +`terminal-pane/__fixtures__/xterm-bypass-event.ts` were byte-identical apart from +an import path, and the `__fixtures__` copy had zero importers — a half-finished +move left in place. The dead copy is deleted and the live one completed its move. + +## G7 — No regression and reviewable comprehensive change + +**Current status: not started.** + +G7 is proven only on the rebased, converged candidate after G0–G6 and all +thirteen journeys are proven. It requires: + +- correctness and security gates; +- A/B input latency and output throughput; +- backpressure and bounded-memory results; +- renderer/app/daemon/relay restore and startup results; +- large-pane, long-session, and multi-host scale results; +- native packaging/startup on macOS, Linux, and Windows; +- WSL, Docker SSH, paired, remote, folder, floating, worktree, drive, and UNC + coverage; +- both live mixed-version directions; +- final categorized LOC census; +- independent repository review with no unresolved P0–P2 findings; +- release-readiness review with no unresolved correctness, security, + compatibility, or performance findings; and +- a detailed comprehensive PR whose claims match the receipts. + +Before implementation, record a performance protocol that makes “no +regression” falsifiable: + +- fixed candidate and baseline builds, hardware, OS, power mode, network shape, + pane/session population, payloads, and background-load policy; +- warm-up policy, randomized A/B order, sample count, raw-data location, and + statistical method; +- input latency, output throughput, memory, backpressure, restore, startup, and + large-pane metrics with directionality; +- deterministic ceilings for writes, scans, queues, listeners, timers, and + allocations on hot paths; and +- a predeclared equivalence/no-regression bound no larger than measured baseline + noise. A nonzero bound handles measurement noise; it is not permission for a + known slowdown and requires explicit user approval. + +Unless an independently reviewed protocol justifies another count, use at least +five warm-up trials and thirty measured trials per latency/startup/restore +configuration, retain raw samples, and report confidence intervals. Throughput, +backpressure, and memory tests must also include a fixed-duration steady-state +run and a leak-slope result. Choose all workloads and thresholds before looking +at candidate results. + +Green CI, thousands of tests, mergeability, or “reviewable for what shipped” do +not prove G7. + +## Thirteen required production journeys + +Every row is currently **not proven**. + +The issue-to-journey matrix in +[`related-open-work.md`](./related-open-work.md#mandatory-issue-to-journey-matrix) +is part of these journeys, not optional context. Every bound incident needs a +red-on-baseline/green-on-candidate oracle, or explicit evidence plus user +acceptance that it is unrelated. + +| # | Journey | Required oracle | +| --: | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Local macOS, Linux, and Windows | The same pane, full binding, and OS process survive renderer and app restart; every stale exact operation is rejected. Run natively on all three OSes. | +| 2 | Daemon and physical WSL | The same PTY survives client and daemon reconnect/restart boundaries; generation skew fails closed without killing or replacing a live successor. | +| 3 | Lazy discovery and skipped-host restart | An unused host is not probed eagerly. After a restart that skipped it, lazy rediscovery restores only that host's sessions and never adopts host-current or sibling-host state. | +| 4 | Concurrent multi-host connections | At least two distinct final-host connections operate simultaneously. One host's disconnect, CAS, timeout, or failure cannot affect the other. | +| 5 | Namespace-partial admission failure | One namespace on a multiplexed connection fails challenge/CAS/grant publication while another commits. Only the failed host-plus-namespace is fenced. | +| 6 | Docker OpenSSH with `MaxSessions=1` | The same remote PID and exact binding survive disconnect and client restart. Authority restart imports exactly or exposes unresolved recovery without creating, killing, or adopting. Explicit close or exact proven teardown durably retires the lease and process; unknown ownership stays visible and recoverable rather than being killed. Count actual remote processes, panes, tabs, bindings, leases, and session-cap slots before and after a settle interval. | +| 7 | Two independent Docker SSH hosts | Two simultaneous final hosts keep endpoint credentials, principals, namespaces, sessions, cursors, failures, and cleanup completely isolated. | +| 8 | Paired client and remote server | The final host remains authoritative across independently updated peers, pairing reconnect, client restart, and remote-runtime restart. | +| 9 | Worktree, folder, floating, drive, and UNC namespaces | Each resolves the same stable host-local namespace across spelling/restart changes, without using client repository ID or target ID as identity. | +| 10 | Stable proof and exact retry | One bounded device proof identity admits fresh process/session nonces. A lost response retries the exact challenge/request; changed, replayed, cross-host, cross-namespace, or host-current state is rejected. | +| 11 | Identity reset and re-enrollment | Crash-resumable host retirement, relay revoke acknowledgement, transport closure, local credential removal, and atomic successor publication occur in order. Offline and old peers remain explicitly pending. | +| 12 | Mixed versions in both directions | No unknown opcode or ungranted publication mutates state. Unsupported challenge, grant, delivery, or operation semantics stay on isolated legacy behavior or fail before mutation. | +| 13 | Performance and scale | Under the predeclared protocol above, input/output latency, throughput, backpressure, memory, reconnect/restore, startup, large-pane, long-session, and multi-host ceilings show no regression against the fixed baseline. Raw samples, confidence intervals, deterministic counters, and leak slopes satisfy their predeclared bounds. | + +## Cross-cutting correctness cases + +Every relevant journey must exercise: + +- stale, missing, unknown, rejected, timed-out, and disconnected evidence; +- concurrent replacement and sibling-host/namespace isolation; +- lost request and lost response; +- cancellation and partial setup cleanup; +- crash immediately before and after every durable boundary; +- restart from disk rather than process-memory state; +- duplicate and out-of-order delivery; +- gaps, overflow, and slow/stalled consumers; +- explicit close versus detach; +- eventual exact retirement after explicit close or proven teardown while + uncertain ownership remains intact and visible; +- legacy data with missing optional fields; +- relay/daemon incarnation reuse; +- sleep/resume and clock movement where timers schedule retries; +- exact remote process/PID census, not only persisted-row state; and +- cleanup that requires positive identity proof. + +## Forbidden proof substitutions + +- A mock is not a native-platform journey. +- A source grep is not a production call-path journey. +- A row marked `expired` is not proof that its remote process exited. +- Optional pane metadata plus an O(n) repair scan is not structural uniqueness. +- An empty list from an unavailable provider is not proof of absence. +- A timer or retry budget is not proof of death. +- A broad test count is not a correctness oracle. +- A test that passes unfixed code is not evidence that a change fixed the bug. +- An open or green PR is not shipped behavior. +- Historical construction receipts are not proof for a rebased candidate. +- Prior art or an uncited LOC comparison cannot delete a requirement. + +## Promotion template + +Before changing any row to `proven`, add a receipt containing: + +```text +Gate or journey: +Candidate SHA: +Merge-base SHA: +Unfixed/red SHA: +Production path exercised: +Exact command or manual protocol: +Environment and versions: +Expected oracle: +Red result: +Green result: +UTC start/end: +Artifact/log: +Independent reviewer: +Unresolved P0/P1/P2: +LOC/performance impact: +``` + +If any field is missing, the row remains partial or not started. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md new file mode 100644 index 00000000000..fce53501a7c --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md @@ -0,0 +1,386 @@ +# New design — goalposts + +Tracks the design approved on 2026-08-09 (`design-explained.html`, detail in +`counsel-design.html`). Updated as work lands. Progress in every status update +is reported against this file. + +## How to read a status + +| Status | Means | +| --- | --- | +| **PROVEN** | An oracle asserts the behaviour, removing the production guard **reddens** that oracle, and the mutation was verified to actually land before the result was believed. | +| **ORACLE RED** | The oracle is written and currently fails for the right reason. The production change is not written yet. This is the intended pre-implementation state. | +| **NOT STARTED** | No oracle. | +| **BLOCKED** | Waiting on another goalpost or an owner decision. | + +A green test is **not** a status. This program shipped three guards that passed +their tests while sitting off the route production takes; "PROVEN" exists to +make that impossible to claim by accident. + +--- + +## Scoreboard + +| | Count | +| --- | --- | +| Step goalposts proven | **7 of 7** (one deleted as unreachable) | +| Global goalposts proven | **5 of 6** — G1 missed, see below | +| Scope removed by evidence | S7 (death rule) — never built | +| Oracle clauses green | 26 | +| Oracle clauses red (awaiting implementation) | 0 | +| Net production lines, correctness work | **+13** | +| Net production lines, incl. the approved affordance | **+83** | +| Net test lines | +2,578 | + +**G1 is not met, and that is reported rather than smoothed over.** The design's +net-negative target assumed the remaining steps were collapses. Three were +(S3 −21, S8 −32, and the relay half of S5), but two genuinely add code: the +single `bindPaneShell` producer (+60, since one function replaces three +divergent call sites without those sites shrinking much) and the D1 +disconnected-pane affordance (+70), which is a new product surface the owner +approved — a feature, not a refactor. Excluding the affordance the correctness +work is +13, i.e. roughly a wash. See "G1" below for the full accounting. + +--- + +## Step goalposts + +### S1 — An identity mismatch is never read as death · **PROVEN** + +*Guarantee.* The relay reports a pane-identity mismatch by saying the pty was +not found. It found it — comparing is how it noticed. That must never reach a +respawn decision as proof of death. + +- Oracle: `src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts`, + `src/renderer/src/components/terminal-pane/reattach-failure-classification.test.ts` +- Clauses: 7 + 3 +- Mutations proven: restoring the destructive wrap reddens 2 clauses; deleting + the classifier guard reddens 1 — clause-selective, each proven separately. +- Landed: `e2524b0472f` + +### S2 — Pane identity is not sent on reattach · **PROVEN** + +*Guarantee.* Moving a pane to another tab must never make its terminal +unreachable. + +- Oracle: same file, `reattach does not ask the relay to police pane identity` (2 clauses) +- Also inverted an existing test that pinned the removed behaviour, so the new + intent stays covered rather than silently dropped. +- Works against **already-deployed relays**: the relay's comparison is + presence-guarded, so not sending the fields disarms it everywhere. No wire + change, no redeploy. +- Landed: `c51be8072ba` + +### S3 — A failed reattach never fabricates an exit · **PROVEN** + +*Guarantee.* On an unproven not-found, the pane is not told the program exited, +ownership is not deleted, provider state is not cleared, and the lease is not +expired. The case routes into the non-destructive recovery branch that already +exists. + +- Oracle: `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts` — 8 green +- Mutation proven: restoring the destructive block reddens 6 of 8 clauses; the + 2 producer pins stay green, so the mutation is clause-selective. +- Two tests pinned the deleted premise ("attach verifies liveness before + answering not-found") and were **inverted**, not patched. +- Shipped with the disconnected-pane affordance below, since panes now stay + visible instead of silently respawning. +- Landed: `c19f1b386b4` (−21 production lines) + +### S4 — One partition per (target, pane) · **PROVEN** + +*Guarantee.* A pane that re-leases under a new relay id across reconnects ends +with exactly **one** live claim; the predecessor is superseded, not left live. + +- Oracle: `src/main/ssh-pane-binding-partition.test.ts` — 7 green (one clause added) +- The highest-value goalpost: the mechanism behind the reported 2 → 19 → 20. +- Mutation proven as a **2×2**, because the two edits mask each other: + + | reader | load fold | result | + | --- | --- | --- | + | hedge (original) | off | 6 red — baseline | + | hedge | off | 4 red — the 2 source-text clauses now green | + | local-only | off | 1 red — the reader carries 3 clauses | + | hedge | **on** | **all green** ← the fold masks the reader | + | local-only | on | all green | + + That fourth row is why an **eighth clause was added**: with the fold shipping, + the fold erases the divergent copy at boot, so the reader guard would have + shipped unproven — precisely the failure mode G5 exists to catch. The new + clause rewrites `ssh:` mid-session (orphan adoption still writes + there) and reddens when the hedge is restored, pinning the reader on its own. +- Five clauses in `ipc/pty.test.ts` pinned the two-partition shape and were + **inverted** to the single home, each keeping an arity check so a re-added + partition argument fails loudly. +- Side effect found while verifying: the renderer never hydrated the `ssh:*` + partition (`listKnownRuntimeHostIds` filters to `runtime:*`), so the Issue + #217 force-quit binding protection had never worked for SSH panes. It does now. +- Landed: `2733c59879b` (+9 production lines — the call sites shrank; the fold + is new state repair) + +### S5 — The superseded-pane fence is live on the reattach path · **PROVEN** + +*Guarantee.* After a relay-driven reattach binds a pane, a stale write aimed at +the superseded predecessor is **refused**. It used to be permitted, because the +fence's bookkeeping was only written by spawn. + +- Oracle: `src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts` — 4 green + (one clause added) +- Collapsed to one `bindPaneShell` producer used by the relay reattach and both + spawn handlers. Error policy stays at the call sites because it genuinely + differs: a caller that just created a shell must clean it up on a failed + durable write; a caller that merely reattached must not detach anything. +- Mutation proven, both sub-guards isolated: + - drop the `rememberPaneKeyForPty` call → 3 clauses redden + - prefer the stored `tabId` over the live layout → 1 clause reddens +- That second clause is **new**. Every pre-existing clause used one tabId on + both sides, so a producer that simply forwarded `lease.tabId` would have gone + green and shipped the moved-pane bug unnoticed. Only the leaf half of a pane + key is remint-stable; `detachTerminalPaneToTab` moves a live pane and its PTY. +- Two source-text clauses were **strengthened**: they used to require the relay + to hold a `persistPtyBinding` call and merely forbade an ssh-partition + argument. The relay now has none, so they assert **zero** direct binding + writes there — a second bind producer is exactly the defect being removed. +- Repaired a latent false green: the "persistence fails" case in + `ssh-relay-session-reconnect-incarnation` was passing because a missing mock + made the call throw a TypeError that happened to emit the asserted + `console.error`. The failure is now injected at the producer. +- Landed: `0f409055d3f` (+60 production lines — the one step that grows) + +### S6 — Settle whether the 30s recovery grant executes at all · **PROVEN — it is dead code** + +*Result.* It cannot execute for a real SSH pane. Verified personally: + +- The lease stores a **relay-native** pty id, normalized on write + (`persistence.ts:7184`, comment: "app ids are global"). +- The caller passes the **app-form** id (`orca-runtime.ts:16478` ← `toAppSshPtyId`). +- The comparison is raw `lease.ptyId === ptyId` (`orca-runtime.ts:6310`) with no + normalization, so the two forms can never be equal. +- The branch is also unreachable for a local pane, since it requires an SSH lease. +- Its covering test seeds both sides as the same literal with a null + connectionId — a shape production cannot produce. + +### S7 — The death rule · **DELETED on S6's evidence** + +Not built. The arbitration machinery existed to referee a branch that never +executes. Per the design: "if E-1 shows the grant branch unreachable, E-2 is +deleted and the marker simply never fires, which is the safe end state." + +### S8 — Remove the dead recovery-grant path · **PROVEN** + +*Guarantee.* Deleting it changes no behaviour, because it has none. + +- Oracle: `src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts`, written + and green **before** the deletion, so inertness is demonstrated rather than assumed. +- It mints **both** id forms from the production helpers rather than as two + hand-typed literals, so it tracks the real namespace split instead of + restating it, and seeds a lease qualifying on every other predicate (state, + worktree, tab, leaf, grace window). Anti-vacuity clauses pin that control + actually reaches the gate rather than bailing out earlier. +- Mutation proven **before** deleting: normalizing the `lease.ptyId === ptyId` + comparison makes the grant fire and reddens the oracle. That is the exact fix + someone would reach for, so the oracle is pinned to unreachability, not to the + throw. +- Deleted: the grant tail, the `terminalPaneRecoveryByIdentity` dedup map (whose + only consumer was the grant), and the dead `ptyId` parameter. +- **Not** deleted, because over-deleting here breaks users: `getRecentExpiredSshLease`, + `hasRecentExpiredSshLeasePane` and `SSH_PANE_RECOVERY_GRACE_MS` all stay. Their + other two callers pass `ptyId` undefined, which short-circuits the broken + comparison — those are live and feed headless-mobile terminal-tab visibility. +- Also **not** done: deleting only the gate while keeping the spawn. That would + have granted a respawn to every disconnected pane — a change in the dangerous + direction. The refusal is what stays. +- Three tests pinned the grant and were **inverted**; a fourth is now + tautological and carries a comment saying so rather than being left hollow. +- Honest limit, recorded in the oracle: unreachable **by construction** for SSH + panes; for a local pane, unreachable only up to a random-UUID collision. +- Landed: `7d44315437a` (−32 production lines) + +### Product affordance (D1) — **SHIPPED with S3** + +A pane that cannot be verified renders as disconnected with two explicit actions +— "Try again" and "Start a new terminal" — instead of silently respawning. + +- No new IPC channel: the silent-respawn decision was always renderer-local. + Both actions are things the code already did, moved behind a click. +- Copy constraint enforced **as an oracle**, not as a review note: the rendered + text must match no death verb and show no wire token. STYLEGUIDE.md:236 already + forbids result verbs without result data, and a failed attach is not result data. +- `TerminalRemoteRuntimeReconnectBanner` → `TerminalPaneDisconnectedBanner`; it + now serves any transport. Existing i18n key strings kept verbatim so no shipped + translation breaks; the SSH copy is additive. +- Landed: `5e1b57bb2f9` (+70 production lines) + +--- + +## Global goalposts + +### G1 — Net production code is negative · **NOT MET (+83 this phase)** + +Counted from the pre-work merge base, production only. Reported as a miss rather +than reframed until it passes. + +| Step | Net production | +| --- | --- | +| S3 — collapse the fabricated exit | **−21** | +| S4 — one partition | +9 | +| S5 — one bind producer | **+60** | +| S8 — delete the dead grant | **−32** | +| D1 affordance (new product surface) | **+70** | +| Banner rename (pure move, no behaviour) | +36 of the above, not real growth | +| **Correctness work only (S3–S8)** | **+13** | +| **Total this phase** | **+83** | + +Why it missed. The target assumed the remaining steps were collapses. Three were. +Two were not, for reasons worth stating: + +- **S5 (+60).** One producer replacing three divergent call sites only shrinks + the code if those sites were duplicating the whole bind. They were not — they + each did *part* of it differently, which is exactly the defect. The producer is + net-new; the call sites shrank by 7 lines. Collapsing the two spawn handlers' + surrounding logic as well would buy the difference back, but it would reorder + side effects that existing ordering assertions pin, so it was left alone. +- **D1 affordance (+70).** A new user-facing surface the owner approved. Features + add lines; counting it against a refactor budget would be the wrong pressure — + the way to make this number go negative would be to delete the affordance. + +Where the remaining slack is, if the goal is to be met later: steps P and K of +the design (the leaf-keyed record replacing `SshRemotePtyLease`, and the lease +readers ported onto it) are the deletions this phase deferred. + +### G2 — No redeploy required for the user-visible fixes · **PROVEN for S1–S8** + +Fixes must work against relays already installed on people's hosts. Every step +that landed is client-side only: + +- S1, S2 — proven previously. +- S3 — deletes client-side reactions to a relay message; the relay is unchanged. +- S4 — client-local persistence layout only; nothing crosses the wire. +- S5 — client-local in-memory fence maps and the durable binding. +- S8 — deletes a client-side branch. +- D1 — renderer only. + +**No relay redeploy is required for any of it**, which matters because the +reported failure is happening on hosts people have already deployed. + +### G3 — Wire compatibility · **SATISFIED — nothing on the wire changed** + +Clients and hosts update independently. This phase added **no** RPC parameter, +no stream opcode and no published field, so there is nothing to negotiate. The +orphan projection that would have added one optional field belongs to step W, +which is not in this phase. + +One wire-adjacent behaviour change is worth naming for reviewers: `terminal.recoverPane` +(RPC) now always refuses for a disconnected pane. The method still exists and its +signature is unchanged, so older clients get a refusal rather than a +`method_not_found` — and the renderer already handles refusal, because that is +what the branch did in practice anyway. + +### G4 — Cross-platform · **SATISFIED by construction** + +macOS, Linux, Windows, WSL. No oracle added in this phase shells out, and none +uses `echo $$` or `ps` — they assert against persisted state, in-memory maps and +rendered text. No rule that landed concludes anything from a relay restart or +from whether a shell survives a hard relay death, which is the Windows-divergent +question the design deliberately refuses to depend on. The affordance copy holds +on every platform precisely because it never claims the shell died. + +### G5 — Every guard is proven live on the production route · **PROVEN for S1–S8** + +For each new guard, an oracle must redden when the **producer** is removed — not +only when the guard itself is removed. + +This exists because it had failed three times here: an inert `mayCreate`, a +keystroke fence inert on reattach, and a respawn gate on a minority path. S5 is +the remediation of the second. + +**It nearly failed a fourth time in this phase, and the rule caught it.** S4's +reader guard (`durablyBoundPtyIdForPane` reading one partition) initially showed +*no* mutation response: restoring the ssh-first hedge left all six clauses green, +because the load fold had already erased the divergent copy at boot. The guard +was correct and would have shipped unproven. The fix was an added clause that +rewrites the ssh partition **mid-session**, which reddens on the hedge and pins +the reader independently of the fold. + +Two further near-misses found and closed the same way: +- S5's clauses all used one tabId on both sides, so "compose the paneKey from + the current tab" was unpinned — a producer forwarding `lease.tabId` would have + gone green. A moved-pane clause now pins it. +- A case in `ssh-relay-session-reconnect-incarnation` was a **latent false + green**: it passed because a missing mock threw a TypeError that happened to + emit the `console.error` it asserted. Now injected at the producer. + +### G6 — No `max-lines` disables · **HOLDING** + +Per AGENTS.md. No per-file bumps either. + +--- + +## Definition of done + +| | Criterion | Status | +| --- | --- | --- | +| 1 | S1–S5 and S7 PROVEN, or S7 deleted on S6's evidence | **MET** — S1–S6 and S8 proven; S7 deleted on S6's evidence | +| 2 | G1–G6 satisfied, each with evidence recorded here | **PARTIAL** — G2–G6 met; **G1 missed at +83**, accounted for above | +| 3 | The reported failures covered by an oracle that reddens without the fix | **MET.** Pane cardinality: S4's ten-reconnect clause, plus an E2E spec driving three real reconnects against a Docker relay (passes). Duplicate resume: the classification clauses, plus an E2E spec where the **host itself** records every shell launch (passes). Writing the second one is what exposed that RC2 was still live — see below | +| 4 | No guard shipped without a producer-side mutation proving it is reachable | **MET** — and it caught one guard that would otherwise have shipped unproven (see G5) | + +### Found after the goalposts were met + +Two defects surfaced *after* every step goalpost was proven, and both were invisible to the unit +oracles that had already gone green. Recorded because they are the most useful evidence in this +phase about where these oracles stop reaching. + +**RC2 survived S3 — found by writing the E2E test.** `7cd7fef927f`. +The reported duplicate agent resume was not fixed. `reattachSshPtySession` maps every relay +not-found to `SSH_SESSION_EXPIRED`, and `isProvenSshSessionGoneError` returned true for it, so the +renderer's reattach arm still respawned. The E2E harness produced the shape directly: a stalled +relay is superseded by a fresh one with no memory of `pty-1`, while the old shells keep running +under the stopped process. The new relay answers not-found for a live shell. + +A not-found proves an exit only if the relay process that minted the pty is the one answering — +design D3 row 2, gated on `relayInstanceId`, which step E-2 never built. `SSH_SESSION_EXPIRED` is +not independent evidence: its only producer is that same mapping, and the token's own doc comment +claimed "the host proved the session is gone", which it never did. + +So `isProvenSshSessionGoneError` returns false. This is also what makes the D1 affordance +load-bearing rather than near-unreachable. The respawn tails are deliberately kept — the design +preserves the grant as a conditional for E-2 — with a clause pinning that nothing reaches them. + +**The fold left half a fence behind — found by adversarial review.** `994733d8b1a`. +`persistPtyBinding` writes the binding and its incarnation into the same partition, and the +incarnation is what its CAS compares. The fold moved only `ptyIdsByLeafId`, so after upgrade the +guard stayed in the partition nothing reads and the CAS compared `undefined` against `undefined`. +Not data loss and not a wrong-shell bind, but a guard silently weakened by a migration is the exact +shape this program keeps finding. + +### Residuals, stated rather than closed quietly + +- **Orphan adoption still writes the `ssh:` partition.** + `adoptTerminalOrphansFromInventory` (via `tryGetWorkspaceSessionHostIdForWorktree`) + writes pane bindings there. S4's reader ignores that partition, so supersession + is unaffected and the guarantee holds — but the write now lands somewhere + nothing reads. Harmless today, and a trap for whoever touches it next. Belongs + with step W. +- **The load fold moves leaf bindings only.** It deliberately does not touch + `tabsByWorktree[*].ptyId` in the ssh partition: that field is a tab-level + pointer no supersession path reads, and nulling it without a local counterpart + to move it to would be data loss. Narrowing the fold to what the defect + actually requires is why `reassignSshTargetId` needed no inversion. +- **The main-side detached branch still has no renderer signal.** S3 made the + main process stop lying; the affordance covers the renderer-initiated reattach + arms. A relay-driven reattach failure remains silent to the user. Closing it + needs a `pty:detached` channel — deferred deliberately, not overlooked. + +## Deliberately out of scope + +- The data-plane work (transport delivery guarantee, binary payloads, one credit + ledger). Real and separately justified, but independent of these goalposts. +- Rebuilding the authority architecture rejected earlier at +60,903 lines. + +## Owner decisions + +| | Decision | Status | +| --- | --- | --- | +| D1 | An unverifiable pane becomes visibly disconnected with two actions, instead of silently respawning | **Approved 2026-08-09** (implicit in approving the design; flagged for correction if not intended) | +| D2 | Whether the older gate/journey framing is rescoped or retired now that this design supersedes it | **Open** | diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md new file mode 100644 index 00000000000..0d20afdf08e --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md @@ -0,0 +1,341 @@ +# Related GitHub work + +This is a curated overlap map, not a merge queue. States were verified against +GitHub on 2026-08-08 UTC and must be refreshed before any rebase, merge, close, +or implementation decision. + +Rules: + +- Do not stack PRs merely because they appear in the same section. +- “Conflict” means semantic or file overlap that requires one coherent design, + not an instruction to merge both. +- A related PR can supply diagnosis, tests, or a small reusable primitive + without becoming a dependency. +- Landing a related PR does not promote G0–G7 or one of the thirteen journeys + unless the final converged tree passes that gate's complete proof. +- GitHub issue/PR text is context, not authority over the user's goal. + +## Immediate incident and competing implementations + +- [Issue #11729 — SSH remote environment not restored after login on Rocky Linux + 9.4](https://github.com/stablyai/orca/issues/11729) — **open**. This is the + named STA-3077 customer incident. Its final acceptance must include Rocky + Linux SSH, repeated reconnect, the same PID/process/session/transcript + identity, restored workspace environment, visible pane count, and actual + remote relay PTY/process count. +- [PR #13111 — stop reconnect grafting panes, stacking leases, and respawning + live shells](https://github.com/stablyai/orca/pull/13111) — **open**, current + branch, green checks, no approval. It is narrow containment and leaves remote + shell reclamation unresolved. One major Docker settle review thread remains + [unresolved](https://github.com/stablyai/orca/pull/13111#discussion_r3739665033). +- [PR #12264 — stop reconnect from grafting ghost terminal + panes](https://github.com/stablyai/orca/pull/12264) — **open and conflicting**. + It targets the same incident and overlaps persistence, PTY, SSH relay, and E2E + files. Mine its diagnosis and discriminating artifacts; do not stack it + wholesale with #13111. +- [PR #12743 — contain reattach to existing panes, quarantine + orphans](https://github.com/stablyai/orca/pull/12743) — **open and + conflicting**. It shares `mayCreate: false` and broader relay coverage, but + quarantine must be judged against the no-reconciliation final design. +- [PR #12456 — reattach on `restoreRequired` instead of expiring a live + session](https://github.com/stablyai/orca/pull/12456) — **open**. It addresses + the #12448 branch through retry rather than cold respawn. Treat it as a + semantic alternative or complementary recovery path only after choosing one + exact restore protocol. +- [PR #9820 — reap orphaned relay PTYs before session-cap + exhaustion](https://github.com/stablyai/orca/pull/9820) — **open**. Its + time-based remote kill conflicts with unknown-is-not-dead and #13111's + deliberate non-kill behavior. Use it as incident evidence, not a safe default + dependency. +- [PR #12798 — opt-in zmx-backed durable SSH + terminals](https://github.com/stablyai/orca/pull/12798) — **open**. This is a + broad architectural alternative for final remote PTY ownership with direct + file overlap. Evaluate capability-by-capability; never blend it into the + current model accidentally. + +## Open incident issues that should become explicit oracles + +- [Issue #9819 — SSH relay leaks orphaned PTYs until the 50-session + cap](https://github.com/stablyai/orca/issues/9819) — explicit close, + disconnected close with a lost shutdown response, and worktree teardown must + durably retry retirement of the exact lease and process. Uncertain sessions + must survive visibly until resolved. Repeated reconnect must reclaim actual + relay PIDs/session slots rather than accumulate toward the cap. +- [Issue #9034 — SSH reconnect repeatedly spawns detached PTYs for one + pane](https://github.com/stablyai/orca/issues/9034) — count actual remote PTYs + and prove bounded one-pane ownership across repeated reconnect. +- [Issue #11006 — transient SSH disconnect treats live relay PTYs as dead and + duplicate-respawns agents](https://github.com/stablyai/orca/issues/11006) — + disconnect with a live process must remain unresolved/reattachable and must + not start another agent. +- [Issue #12447 — closed SSH tabs resurrect and auto-resume old + sessions](https://github.com/stablyai/orca/issues/12447) — explicit close must + persist its pending tombstone before a kill RPC, survive a lost reply plus app + and relay restart, retry after handshake, reclaim the exact remote PTY, and + never recreate UI or automatic resume. +- [Issue #12448 — `restoreRequired` is misreported as session + expiry](https://github.com/stablyai/orca/issues/12448) — source recovery must + replay/repaint on the same PID and binding without starting a replacement + `--resume`; identity drift, timeout, or unknown state must not become proof of + process death or cold resume authority. +- [Issue #12699 — SSH sleeping-agent wake sweep forks live remote + sessions](https://github.com/stablyai/orca/issues/12699) — all wake/resume + paths must share the same exact liveness and identity contract. +- [Issue #12683 — remote-runtime disconnect spends recovery on stale same-handle + reattach](https://github.com/stablyai/orca/issues/12683) — paired/web terminal + recovery must fence generations and retain automatic liveness. +- [Issue #10208 — duplicate terminal tab on worktree session + restore](https://github.com/stablyai/orca/issues/10208) — local/worktree + restore must prove the same pane and process rather than creating a duplicate. + +These issues are separate customer-visible symptoms. A single structural model +should make their shared invalid transitions impossible, while each issue keeps +its own discriminating regression journey. + +## Daemon, relay generation, and liveness work + +The acceptance matrix must also reproduce these open lifecycle incidents: + +- [Issue #11904 — tab close with a broken binding orphans the daemon + session](https://github.com/stablyai/orca/issues/11904) — close intent must be + durable before transport work and eventually retire the exact process. +- [Issue #8585 — detached relay generations are never + reaped](https://github.com/stablyai/orca/issues/8585) — termination must name + the exact relay generation; path/PID reuse or a failed connection cannot kill + a successor. +- [Issue #9138 — updates leave old daemon generations and sessions + alive](https://github.com/stablyai/orca/issues/9138) and + [issue #11342 — stale daemon/PTY generations leak across upgrade and + close](https://github.com/stablyai/orca/issues/11342) — upgrade must converge + without invisible sessions, guessed ownership, or unbounded retention. +- [Issue #10415 — Windows daemon crash on unkillable PTY and protocol + bump](https://github.com/stablyai/orca/issues/10415) — native Windows must + preserve live old-generation sessions or expose exact unresolved recovery; + one unkillable PTY cannot crash the daemon. +- [Issue #8275 — worktree teardown kills unrelated split-pane + sessions](https://github.com/stablyai/orca/issues/8275) — teardown and failure + must remain exact-namespace/exact-binding scoped. + +- [PR #12882 — publisher-owned daemon endpoint + replacement](https://github.com/stablyai/orca/pull/12882) — **open and + mergeable** when checked. It provides single-writer endpoint publication and + three-valued liveness. Reconcile it before editing overlapping daemon paths; + do not assume the older handoff's merge order is still current. +- [PR #11622 — fence daemon session ownership across + generations](https://github.com/stablyai/orca/pull/11622) — **open** and held + behind further audit per its description. Strong explicit + owned/unavailable/ambiguous semantics, but broad overlap makes it design input, + not an automatic dependency. +- [PR #9833 — report local PTY inventory + readiness](https://github.com/stablyai/orca/pull/9833) — **open**. Its + pending/ready/stale distinction is reusable context for unknown-is-not-dead. +- [PR #12760 — reap only the proven relay + generation](https://github.com/stablyai/orca/pull/12760) — **open**. It adds + exact relay generation ownership before termination and conflicts in + `ssh-relay-session.ts`; reconcile before designing remote shell cleanup. +- [PR #8618 — reap a detached relay after failed + reconnect](https://github.com/stablyai/orca/pull/8618) — **open** and an older + competing predecessor to #12760. Do not combine both ownership schemes. +- [PR #12749 — wait for a disconnected PTY owner's full + grace](https://github.com/stablyai/orca/pull/12749) — **open**. Useful owner + admission and grace context; timers may schedule retry but cannot independently + authorize takeover or destruction. +- [PR #12702 — stop sleeping-agent wake sweep from forking live SSH + sessions](https://github.com/stablyai/orca/pull/12702) — **open**. Complementary + #12699 work that must converge on the same ownership/liveness behavior; it + need not share an internal primitive unless the reconciled design calls for + one. +- [PR #10118 — durable persistence write + seam](https://github.com/stablyai/orca/pull/10118) — **open**. It overlaps + `persistence.ts` and is relevant to durability and main-thread performance; + reconcile before large persistence changes. + +## Paired runtime, remote server, and E2EE lifecycle work + +These open issues are mandatory inputs to the paired/remote/platform journeys: + +- [Issue #11495 — paired viewer loses attachments across update/restart while + host PTYs remain alive](https://github.com/stablyai/orca/issues/11495) — the + reported large host population is a strong restore/scale oracle; each pane + must return to the same host PTY/process. +- [Issue #11265 — paired terminals stall despite live TCP + sockets](https://github.com/stablyai/orca/issues/11265) — transport connectivity + is not delivery liveness; recovery must resume output without an unrelated RPC + kick. +- [Issue #11803 — remote stop respawns tabs from a stale + owner](https://github.com/stablyai/orca/issues/11803) — exact durable retirement + must remain retired after host/client restart. +- [Issue #12241 — paired-host session partitions grow without + bound](https://github.com/stablyai/orca/issues/12241) — paired and scale + journeys must prove bounded state and safe lost-consumer + retirement/compaction liveness. +- [Issue #11574 — same-server re-pairing changes host identity and strands + sessions](https://github.com/stablyai/orca/issues/11574) — stable final-host + identity must survive re-pairing without adopting another host. +- [Issue #9827 — investigate WSL session + restoration](https://github.com/stablyai/orca/issues/9827) and + [issue #11339 — WSL commands are not restored after + restart](https://github.com/stablyai/orca/issues/11339) — the daemon/WSL + journey must run on physical WSL and preserve both shell commands and agent + sessions. + +Additional open incident inputs must be triaged into a named journey or ruled +unrelated with explicit evidence and user acceptance: + +| Issue | Required question | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| [#11800 — paired null-PTY terminal ghosts](https://github.com/stablyai/orca/issues/11800) | Does host topology advancement retire exact placeholders without inference or a permanent timer reconciler? | +| [#9092 — remote runtime fails after macOS sleep](https://github.com/stablyai/orca/issues/9092) | Does generation-fenced reconnect recover after sleep with bounded resources and no identity change? | +| [#9585 — remote ghost tabs after host restart](https://github.com/stablyai/orca/issues/9585) | Are exit/retirement outcomes durable and idempotent across host restart? | +| [#9562 — remote host session restarts empty](https://github.com/stablyai/orca/issues/9562) | Does the complete boundary plus replay restore terminal state after server restart? | +| [#12568 — no PTY provider after SSH relay recovery/update](https://github.com/stablyai/orca/issues/12568) | Does lazy rediscovery restore the exact host/provider generation? | +| [#10385 — mobile remains falsely connected after RPC stalls](https://github.com/stablyai/orca/issues/10385) | Can E2EE/mobile liveness recover when transport looks connected but RPC is stalled? | +| [#12140 — mobile pairing socket closes immediately](https://github.com/stablyai/orca/issues/12140) | Do capability, identity, and E2EE negotiation fail explicitly and recover across LAN/Tailscale? | +| [#8129 — mobile events lost while disconnected](https://github.com/stablyai/orca/issues/8129) | Are semantic outcomes durable and replayed to a disconnected/backgrounded consumer? | + +- [PR #11575 — preserve host identity across same-server + re-pairing](https://github.com/stablyai/orca/pull/11575) — **open**. Direct + identity context for the paired reconnect journey. +- [PR #12768 — restore paired snapshots at the source + grid](https://github.com/stablyai/orca/pull/12768) — **open**. Complementary + host-snapshot authority work with renderer file overlap. +- [PR #9093 — recover remote runtime streams after client + wake](https://github.com/stablyai/orca/pull/9093) — **open**. Provides + generation-cancelled bounded-backoff reconnect context. +- [PR #10235 — self-heal remote subscriptions after server + restart](https://github.com/stablyai/orca/pull/10235) — **open**. Relevant to + E2EE session loss hidden by a live relay transport. +- [PR #11596 — recover falsely connected mobile RPC + sessions](https://github.com/stablyai/orca/pull/11596) — **open and + conflicting**. Broad mobile/E2EE reconnect context; do not absorb its parallel + lifecycle machinery without a full state-machine audit. +- [PR #12036 — stop blocking E2EE handshake on synchronous ACL + writes](https://github.com/stablyai/orca/pull/12036) — **open and conflicting**. + Preserve its metadata-versus-correctness durability distinction and measured + Windows performance constraint during identity lifecycle work. +- [PR #12134 — refresh stale capabilities before + pairing](https://github.com/stablyai/orca/pull/12134) — **open**. Capability + freshness and reauthentication are required mixed-version/pairing context. +- [PR #11804 — make remote stop retirement + durable](https://github.com/stablyai/orca/pull/11804) — **open**. Strong + complementary evidence for exact persisted owner routing and host-side + retirement acknowledgement. +- [PR #12987 — distinguish an absent worktree from an empty tab + list](https://github.com/stablyai/orca/pull/12987) — **open**. Reusable + unknown-versus-authoritative-empty semantics. +- [PR #12751 — preserve tabs when a snapshot reports + none](https://github.com/stablyai/orca/pull/12751) — **open** and a competing + less-exact alternative to #12987; its described multi-client non-convergence + must not become the final model. +- [PR #12339 — stop remote workspace pulls from resurrecting closed + tabs](https://github.com/stablyai/orca/pull/12339) — **open**. Direct topology + reconciliation context for explicit close and stale snapshots. +- [PR #13013 — stop stale snapshot applies from erasing fresh terminal + tabs](https://github.com/stablyai/orca/pull/13013) — **open**. Relevant legacy + snapshot race; decide whether its machinery is deleted by the final authority + cutover rather than layering another permanent reconciler. +- [PR #12903 — retry snapshot tabs whose paths resolve after + apply](https://github.com/stablyai/orca/pull/12903) — **open**. Deferred + hydration is relevant legacy-importer context and may become deletable after + authoritative namespace resolution; do not retain it by default. + +## Delivery, backpressure, and lifecycle alternatives + +- [Draft PR #12220 — keep SSH alive when a terminal outruns a slow + link](https://github.com/stablyai/orca/pull/12220) — **open draft**. Useful G3 + and performance evidence, but its own description has behavioral blockers and + a stalled-consumer wedge. Do not treat it as merge-ready. +- [PR #10744 — durable lost-worker archive and mixed-version SSH + revive](https://github.com/stablyai/orca/pull/10744) — **open** and broad. It + is major architectural context for exact close, outcomes, retirement, and + mixed versions, with substantial overlap. Mine proof and semantics rather + than stacking blindly. +- [PR #11801 — prune stale remote tab + ghosts](https://github.com/stablyai/orca/pull/11801) — **open**. Its snapshot + omission plus timer policy is relevant but may conflict with the rule that + absence/time alone cannot prove destructive cleanup. + +## Mandatory issue-to-journey matrix + +This table binds customer incidents to the thirteen journeys in +[`goalposts.md`](./goalposts.md#thirteen-required-production-journeys). Closing +an issue or merging a narrow PR does not remove its oracle. Each row must be red +on an unfixed baseline and green on the final candidate, or the user must accept +explicit evidence that the issue is unrelated. + +| Open issue(s) | Journey(s) | Mandatory discriminating outcome | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| [#11729](https://github.com/stablyai/orca/issues/11729) | 6, 9, 13 | Rocky Linux reconnect preserves the same workspace, transcript, pane, binding, PID, and environment without PTY growth. | +| [#9819](https://github.com/stablyai/orca/issues/9819), [#9034](https://github.com/stablyai/orca/issues/9034) | 6, 13 | Reconnect cannot accumulate detached PTYs/session slots; explicit close and teardown reclaim the exact process, including lost-response retry. | +| [#11006](https://github.com/stablyai/orca/issues/11006), [#12448](https://github.com/stablyai/orca/issues/12448), [#12699](https://github.com/stablyai/orca/issues/12699) | 2, 6 | Disconnect, source recovery, and wake keep the same live PID/session and never start a duplicate resume without proof of death. | +| [#12447](https://github.com/stablyai/orca/issues/12447), [#11904](https://github.com/stablyai/orca/issues/11904) | 2, 6 | Durable close intent survives lost RPC, app/daemon/relay restart, retries exact retirement, reclaims the process, and never resurrects UI. | +| [#10208](https://github.com/stablyai/orca/issues/10208) | 1, 9 | Worktree switch/restore preserves one exact pane/process and does not duplicate a tab. | +| [#8585](https://github.com/stablyai/orca/issues/8585) | 2, 6, 12 | Cleanup affects only the proven relay generation across reconnect and version change; ID/path reuse cannot kill a successor. | +| [#9138](https://github.com/stablyai/orca/issues/9138), [#11342](https://github.com/stablyai/orca/issues/11342) | 2, 12, 13 | Upgrade/restart converges old daemon generations and their PTYs without invisible sessions, leaks, or destructive guessing. | +| [#8275](https://github.com/stablyai/orca/issues/8275) | 1, 2, 5 | Worktree teardown cannot terminate a sibling namespace or split-pane session. | +| [#10415](https://github.com/stablyai/orca/issues/10415) | 1, 2, 5, 12 | Native Windows handles an unkillable PTY and protocol bump without daemon crash, live-session loss, or cross-session damage. | +| [#9827](https://github.com/stablyai/orca/issues/9827), [#11339](https://github.com/stablyai/orca/issues/11339) | 2, 9 | Physical WSL restart restores the same shell command and agent session under the exact namespace/binding. | +| [#12568](https://github.com/stablyai/orca/issues/12568) | 3, 6, 12 | Relay recovery/update lazily rediscovers the same SSH host/provider and restores only its exact sessions. | +| [#11495](https://github.com/stablyai/orca/issues/11495), [#11574](https://github.com/stablyai/orca/issues/11574) | 4, 8, 10, 12, 13 | Paired restart/re-pair preserves stable host identity and every reported live host PTY at scale across independent updates. | +| [#11265](https://github.com/stablyai/orca/issues/11265), [#9092](https://github.com/stablyai/orca/issues/9092) | 8, 13 | Live-but-stalled transport and sleep/wake automatically resume bounded delivery without an unrelated RPC kick or identity replacement. | +| [#11803](https://github.com/stablyai/orca/issues/11803), [#9585](https://github.com/stablyai/orca/issues/9585), [#11800](https://github.com/stablyai/orca/issues/11800) | 3, 8 | Remote stop/host topology outcome remains durably retired after restart; no stale-owner tab or null-PTY ghost returns. | +| [#9562](https://github.com/stablyai/orca/issues/9562), [#8129](https://github.com/stablyai/orca/issues/8129) | 3, 8, 13 | Complete boundary plus replay restores disconnected/restarted consumers without an empty session or lost semantic outcome. | +| [#12241](https://github.com/stablyai/orca/issues/12241) | 8, 11, 13 | Paired partitions and retained-consumer state remain bounded; lost-consumer retirement safely restores compaction liveness. | +| [#12683](https://github.com/stablyai/orca/issues/12683) | 8, 12 | Stale same-handle reconnect is generation-fenced and automatic recovery remains live across independent peer versions. | +| [#10385](https://github.com/stablyai/orca/issues/10385), [#12140](https://github.com/stablyai/orca/issues/12140) | 8, 10, 12 | False-connected or rejected E2EE/pairing state fails explicitly and reauthenticates without stale identity, capability, or session adoption. | + +Journeys 4, 5, 7, 10, and 11 remain mandatory even where no single issue fully +specifies them. This matrix adds incident oracles; it never narrows the journey +definitions. + +## Original containment/refactor line + +- [PR #12600 — break Activity and terminal React 185 + loops](https://github.com/stablyai/orca/pull/12600) — **merged**. It is current + containment/background context, not the whole program and not a release gate. +- [PR #12634 — replace React 185 timers with structural + handoffs](https://github.com/stablyai/orca/pull/12634) — **open**. This is the + direct structural follow-up for deleting #12600's window/timer brakes and must + be reconciled with the broader terminal-session model. +- [PR #13110 — sandboxed preload artifact + guard](https://github.com/stablyai/orca/pull/13110) — **open**, independent of + #13111, green at its head, no approval. Its unresolved major review says raw + source matching can reject safe strings/comments; resolve or supersede that + [finding](https://github.com/stablyai/orca/pull/13110#discussion_r3739646949) + before calling it ready. + +## Already merged base context + +These were described as open in older notes but were merged when this handoff +was written: + +- [PR #12474 — isolate same-path folder workspace PTY + identity](https://github.com/stablyai/orca/pull/12474) +- [PR #12477 — stop a closed remote workspace window reading + Ready](https://github.com/stablyai/orca/pull/12477) +- [PR #12600 — React/terminal loop containment](https://github.com/stablyai/orca/pull/12600) + +Refresh `main` and recalculate conflicts; do not follow the stale sequence +`#12882 → #12474 → #12477 → #13111` as written. + +## Required GitHub refresh before action + +At minimum, re-query: + +```bash +gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +gh pr view 13110 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +gh issue view 11729 --repo stablyai/orca --json state,title,updatedAt,url +``` + +Refresh the complete catalog in one batched request: + +```bash +handoff_query_text="$(tr '\n' ' ' < docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql)" +gh api graphql -f query="$handoff_query_text" +``` + +The query includes current PR state/check summaries and unresolved-thread state +for #13110/#13111. Read PR bodies/diffs only for the slice being changed. Do not +close, comment, merge, push, or checkout a PR from the handoff audit without +explicit authorization. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html new file mode 100644 index 00000000000..4497d1f42e4 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html @@ -0,0 +1,376 @@ + + + + + + + Terminal Session Correctness — Design & Test Report + + + +
+
+

Terminal Session Correctness

+

Design & test report · branch nwparker/sta-3077-reattach-pane-cardinality · PRs #13110, #13111

+
+ +
+
+214
Production lines
+
+60,903
Rejected approach
+
12
Oracles that bite
+
2 / 13
Journeys proven
+
0 / 8
Gates proven
+
2
Claims retracted
+
+ +
+

The one-sentence version

+

Every bug here is the same bug — identity compared with the wrong key, or not compared at all. A previous attempt added a second identity system beside the first at a cost of 60,903 lines and fixed none of the three root causes. This fixes all three in 214.

+
+ +

1 · What sits behind one terminal

+

Five separate things line up behind the rectangle you see.

+
+
Anatomy of a terminal⤢ zoom
+

The real computer

Orca's bookkeeping

What you see

bound to

points at

claims

runs on

Pane
a rectangle in a tab

Binding
this pane uses that shell

Lease
we own a remote shell

PTY
the shell process

Host
your Mac, or an SSH box

The real computer

Orca's bookkeeping

What you see

bound to

points at

claims

runs on

Pane
a rectangle in a tab

Binding
this pane uses that shell

Lease
we own a remote shell

PTY
the shell process

Host
your Mac, or an SSH box

+
+

The PTY is the real shell doing the work. The binding is the note connecting it to your pane. On a remote machine a lease is a second note saying “we still own that shell.” Every failure below is one of those notes pointing at the wrong thing.

+ +

2 · The reported failure

+
+
Reconnect growth — a real customer report⤢ zoom
+

2 terminals

network hiccup

reconnect

19 terminals

hiccup

20 terminals
most are ghosts

2 terminals

network hiccup

reconnect

19 terminals

hiccup

20 terminals
most are ghosts

+
+

The remote machine also fills with unused shells until it refuses to start more. A second, worse variant: an AI coding agent resumed twice — two claude processes writing to one conversation file. One report reached five.

+ +

3 · Three root causes

+ +

Cause 1 — the lease forgot its pane

+

A lease identified a shell but never checked which pane it belonged to. Every reconnect added a note; nothing removed one.

+
+
Lease accumulation⤢ zoom
+

reconnect

reconnect

Pane A

lease → shell-1

lease → shell-2

lease → shell-3

nothing retires
the old notes

next reconnect restores
all three

reconnect

reconnect

Pane A

lease → shell-1

lease → shell-2

lease → shell-3

nothing retires
the old notes

next reconnect restores
all three

+
+

FIXED A lease is identified by its pane. One pane, one live lease. Superseded leases are marked expired, not killed — the remote shell keeps running, because losing a note is not proof the shell died.

+ +

Cause 2 — reconnecting could create panes

+
+
Reattach decision⤢ zoom
+

yes

no · before

no · after

reconnect finds a lease

does its pane
still exist?

reattach — correct

invent the pane
👻 ghost appears

refuse
shell left running, unbound

yes

no · before

no · after

reconnect finds a lease

does its pane
still exist?

reattach — correct

invent the pane
👻 ghost appears

refuse
shell left running, unbound

+
+

FIXED Reattach binds only. Opening a new terminal still creates, exactly as before.

+ +

Cause 3 — a living shell reported as dead

+ + + + + + +
Real meaningReported asConsequence
This shell is goneSESSION_EXPIREDcorrect
Shell is fine — output pipe needs re-establishingSESSION_EXPIREDrespawn → agent resumed twice
+

FIXED The second case has its own message, and respawn now requires positive proof. Anything unrecognised is “unknown”, which never justifies a replacement.

+ +

4 · The rule underneath all of it

+
+
Unknown is not dead⤢ zoom
+

shell exited

can't reach it

timed out

absent from a list

3 failed retries

evidence

what does it prove?

DEAD
cleanup allowed

UNKNOWN

nothing destructive
wait · retry · ask

shell exited

can't reach it

timed out

absent from a list

3 failed retries

evidence

what does it prove?

DEAD
cleanup allowed

UNKNOWN

nothing destructive
wait · retry · ask

+
+

The liveness check also became three-valued. It previously returned yes/no, so a provider whose socket was down could only answer “no” — meaning dead — when the honest answer was “I can’t see”.

+ +

5 · What changed

+ + + + + + + + + + +
#ChangePrevents
1Reattach binds, never createsGhost panes
2Leases identified by pane2 → 19 → 20 growth
3Heal already-corrupted lease dataExisting installs stuck at 20
4“Needs re-establishing” ≠ “expired”Duplicate agent resume
5Liveness can answer “unknown”Live shells declared dead
6One PTY’s failure can’t drop the shared channelOne bad pane killing every session on a host
+
+
Change 6 — blast radius⤢ zoom
+

after

pane 3 can't re-prove
its output stream

park pane 3 only

its shell keeps running
siblings untouched

before

pane 3 can't re-prove
its output stream

give up after N tries

drop the whole
host connection

every pane, file transfer
and git command dies

after

pane 3 can't re-prove
its output stream

park pane 3 only

its shell keeps running
siblings untouched

before

pane 3 can't re-prove
its output stream

give up after N tries

drop the whole
host connection

every pane, file transfer
and git command dies

+
+
+

No wire change

+

The keystroke guard adds nothing to the payload. Orca already keeps two lookup tables in lock-step, so when they disagree that is proof the id is stale — a purely server-side check.

+
+ +

6 · How we decide a test is worth trusting

+
+

This happened to us twice

+

An end-to-end test was reported as proving the reconnect fix. It passed with the fix and with it removed — retracted. Separately, a guard shipped with no production caller at all; every test stayed green because they called the capability directly.

+
+
+
The four-step proof⤢ zoom
+

stays green

1 · run it
green

2 · delete a guard
in production code

3 · run it
MUST go red

4 · restore
green again

worthless
say so — don't ship
it as proof

stays green

1 · run it
green

2 · delete a guard
in production code

3 · run it
MUST go red

4 · restore
green again

worthless
say so — don't ship
it as proof

+
+

Delete a real guard, not the test. Prefer one clause at a time — a mutation reddening exactly one claim proves the claims independently. Watch it fail yourself: six of twelve oracles were personally re-verified, and two of those re-checks overturned the original claim.

+ +

7 · Why “same id” is not good enough

+

The most valuable lesson here. Break the quit path so it destroys shells instead of detaching, then restart:

+ 
tab id    same ✓
+pane id   same ✓
+pty id    same ✓
+OS pid    13756 → 8852   ✗
+start     02:01:29.337 → 02:01:34.728   ✗
+
+Every id-only restart test stays green here.
+Your shell is gone.
+
+
Proving the process really survived⤢ zoom
+
KernelShellOrcaTestKernelShellOrcaTestrestart the apptype "echo MARKER=$$"through the real write pathMARKER=13756start time of 13756?02:01:29.337ask againMARKER=13756start time of 13756?02:01:29.337 — same process ✓
KernelShellOrcaTestKernelShellOrcaTestrestart the apptype "echo MARKER=$$"through the real write pathMARKER=13756start time of 13756?02:01:29.337ask againMARKER=13756start time of 13756?02:01:29.337 — same process ✓
+
+ +

8 · Journey status

+ + + + + + + + + + + + +
#JourneyStateWhy
1Local restart — macOS · Linux · WindowsPROVENClause-selective on all three platforms
2Daemon & physical WSLPROVENClause-selective on macOS, Linux, WSL2
4Concurrent multi-hostBY CONSTRUCTIONA mux is per-host — dispose cannot cross hosts, so there is nothing to break
3Lazy discoveryNO SELECTIVE MUTATIONRemoving the real guard breaks setup before the clause is reached
6Docker MaxSessions=1PARTIAL2 clauses redden; a third survived four guard removals
12Mixed versionsWITHDRAWNThe token never crosses a version boundary in production
13Performance & scaleIN PROGRESS1 of 10 dimensions measured; rest running
5, 10, 11Device proof · identity reset · namespace admissionNO SUBJECTThe code does not exist. No oracle is writable at any effort.
+
+
Why a property may resist proof⤢ zoom
+

yes

no — true by
construction

no code exists

property to prove

is a shipped guard
protecting it?

remove it → red
provable

nothing to remove
not provable by mutation

nothing to test
not writable

yes

no — true by
construction

no code exists

property to prove

is a shipped guard
protecting it?

remove it → red
provable

nothing to remove
not provable by mutation

nothing to test
not writable

+
+
+

Read this carefully

+

“True by construction” is a good outcome — the property holds because of how the code is shaped, not because a guard watches it. It simply cannot be demonstrated by breaking something. Reporting it as proven would be false.

+
+ +

9 · Traps that can fake a result

+ + + + + + + + +
TrapEffectMitigation
Shared temp directory — the harness writes a pointer to a machine-wide pathConcurrent runs can fake a red and hide a real oneIsolated TMPDIR per run
Stale build — e2e runs a built appYou tested the old codeCheck build timestamp against source edit
Mutation silently didn’t applyTest stays green, looks like “no teeth” — wrong conclusionVerify the mutation landed before believing the result
Serial mode reports “did not run”A skip mistaken for a passRe-run the others alone under the same mutation
+

Two of our own tests were broken in ways only real hardware revealed: one failed on every Linux run (a single unpolled read where its sibling polls for 15s), and one could not execute at all on Windows (echo $$ and ps don’t exist there).

+ +

10 · Open decisions

+ + + + + + + + +
IDDecisionBlocks
D2Re-scope G4? It demands an architecture where 0 of 394 files exist, measured at +62k LOC — the approach that already failed at 0/13.G4, G1, G3
D3Strike or defer J5 / J10 / J11? They have no production subject.3 journeys
D4Architectural gates need a “reviewed diff + file list” receipt; the doc’s red-SHA template makes them structurally unpromotable.G0, G1, G4
D5The input quarantine is load-bearing, not superseded. Deleting it re-opens a verified rm -rf hazard. Amend the clause, or accept something weaker on remote hosts?G6
+
+

The hazard, verified

+

Disable the quarantine and the oracle reproduces it exactly: echo hi; rm -rf x arrives as cho hi; rm -rf x. The shell rejects cho and then runs rm -rf x. The replacement was built and costed at roughly +140 lines to delete 88 — and it does not work on remote hosts at all, because old hosts never publish the incarnation it needs.

+
+ +

11 · Corrections made along the way

+

Included because a report without them is not trustworthy.

+ + + + + + + + + + +
ClaimOutcome
“The reconnect e2e proves the fix”Retracted — passed with the fix removed, twice, isolated
“Cross-version tests confirm old clients are safe”Withdrawn — they exercise a route production never takes
“mayCreate fixes reattach”Was inert — no production caller for several commits
“A branded binding type is the fix”Rejected — three claims false; would have added a second identity system
“Fixtures compile as production”Corrected — none ship; the sweep was stopped
“bash 5.3.9 causes the WSL failure”Corrected — macOS runs the same version and passes
+ +
+ Generated for review alongside design-overview.md, test-overview.md and goalposts.md. + Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
+ × +
+
scroll to zoom · drag to pan · Esc to close
+
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md new file mode 100644 index 00000000000..62c4274d5ea --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md @@ -0,0 +1,275 @@ +# Resume plan and validation record + +This plan preserves the comprehensive objective while allowing independently +safe stacks. It does not require reviving the 814-path construction snapshot. + +## First safe commands + +Run from the primary worktree before editing: + +```bash +pwd +git status --short --branch --untracked-files=all +git rev-parse HEAD origin/main +git rev-list --left-right --count HEAD...origin/main +git branch --list 'nwparker/react185-authority-snapshot' +git show --stat --oneline fddb19f6977 +gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +``` + +Expected handoff checkpoint before other changes: + +- branch `nwparker/sta-3077-reattach-pane-cardinality`; +- HEAD `5369479be2953f45cea9ab5cfcde756dd9660548`; +- snapshot branch and commit present; +- only this handoff folder and its `.gitignore` allow-list are new handoff work. + +If the branch, HEAD, or dirty paths differ, inventory and preserve them before +continuing. Never reset, clean, overwrite, or switch away from unknown user work. + +## Phase 0 — Reconcile reality before design or code + +1. Refresh `origin/main` with `git fetch origin main`—never `git pull`—and + refresh the live state/reviews of every PR in + [`related-open-work.md`](./related-open-work.md) that touches the intended + slice. +2. Recalculate the candidate merge base, categorized LOC, changed-file overlap, + and failing/obsolete tests. +3. Freeze the program accounting baseline before implementation. The default is + `5ed45739e94bdf6460364e033bfcec9b32c0b42a` (the base of #12600). Build an + aggregate carry-in ledger for every program-attributable PR already merged + after it; do not measure only the last PR in a stack. +4. Read the final diff of #13111, #12264, and #12743 with `gh pr view` and + `gh pr diff`, or in disposable isolated worktrees. Never use `gh pr checkout` + or switch branches in the primary worktree. Select or design one coherent + lease/reattach model; do not combine mutually inconsistent quarantine, kill, + and non-kill policies. +5. Reconcile #12882 before overlapping daemon-endpoint integration and #12760 + before overlapping relay-generation cleanup. Neither blocks unrelated SSH + proof work by declaration. +6. Treat the preserved snapshot as a local-only mine, not a branch to finish. + Inspect it with `git show fddb19f6977:` or in a disposable isolated + worktree; never switch the primary worktree to it. + +Decision output required before implementation: + +- exact incident mechanisms being eliminated; +- exact authority/identity semantics retained or changed; +- explicit remote-shell lifecycle and user recovery behavior; +- explicit lost-consumer liveness/compaction behavior; +- PR/stack boundaries and dependency order; +- production modules expected to be deleted; and +- tests that will discriminate the baseline from the candidate; and +- a predeclared performance protocol with fixed workloads, samples, + statistical/noise bounds, deterministic counters, and raw-data locations. + +## Phase 1 — Build discriminating journeys first + +Before relying on a code change, create or repair oracles that reproduce the +failure on an unfixed baseline. + +### STA-3077 Docker SSH oracle + +The Docker journey must: + +- run a real OpenSSH server with `MaxSessions=1` verified from the server; +- create a known pane and record tab, leaf, binding, relay PTY, and remote PID; +- seed or naturally reproduce duplicate/stale lease state from the field; +- exercise disconnect, client restart, reconnect, and authority/relay restart; +- wait an explicit settle interval after successful reattach; +- recensus visible panes, tabs, exact bindings, persisted leases, relay PTYs, + and actual remote processes/PIDs; +- prove no pane or shell is created, rebound, lost, or silently killed; +- prove explicit unresolved recovery for state that cannot be established; and +- exercise explicit close, lost shutdown response, reconnect retry, and + worktree teardown; prove exact remote PID/session-slot reclamation while an + uncertain sibling survives; and +- fail for the intended reason on the unfixed baseline. + +A clean reconnect that passes on both revisions is not this oracle. + +### Duplicate-agent oracle family + +Cover transient SSH loss, `restoreRequired`, sleeping-agent wake, worktree +switch, and closed-tab restart. Assert process count and transcript/session +ownership, not just pane/store state. No path may start a second `--resume` +process while the first process is live or unknown. + +### Daemon and remote oracles + +Cover unknown inventory, endpoint handover, daemon restart, relay generation +reuse, client sleep/wake, remote-runtime restart, and stale subscription repair. +A retry limit may end an attempt or surface a failure; it may not itself prove a +session dead or authorize destructive replacement. + +## Phase 2 — Reconcile and approve the long-term design + +The implementation design must explicitly settle: + +- final-host identity versus client routing aliases; +- canonical host-local namespace for worktree, folder, floating, drive, and UNC + forms; +- structural one-active-binding/lease ownership rather than optional metadata + plus repair scans; +- exact operation fencing across every transport; +- explicit detach, close, retirement, orphan recovery, and process-reclamation + operations; +- durable outcome/replay semantics across app, renderer, host, and transport + restarts; +- mixed-version capability and isolated legacy behavior; +- lost-device consumer retirement and compaction liveness; +- one-way migration/cutover; and +- the four-role deletion oracle and minimised, justified production-code target. + +Reuse existing pane, PTY, incarnation, CAS, and three-valued liveness primitives +where their semantics match. Do not equate client routing IDs with host identity +or namespace without proving the full semantics. + +If the chosen design differs from the preserved normative design, write an +explicit mapping for every invariant and every journey, identify what changes, +show equal or stronger safety/liveness, and obtain the user's approval. A +handoff author or implementation agent cannot approve that scope change alone. + +## Phase 3 — Implement as a convergent stack + +Stacks are allowed for reviewability. Temporary production stopgaps are not the +end state. + +A sensible dependency shape is: + +1. discriminating journey harnesses and behavior oracles; +2. final shared identity, liveness, and exact-binding primitives; +3. local/daemon/WSL/SSH/relay/paired/remote adapters using those primitives; +4. durable outcome, projection, reset, and recovery lifecycle; +5. one-way migration and capability cutover; +6. deletion of superseded writers, ledgers, repair scans, timers used as verdicts, + and duplicate state machines; retain `terminal-input-quarantine.ts` until a + replacement passes its destructive-input oracle; and +7. full platform, skew, performance, packaging, and independent review proof. + +Each stack layer must be safe and tested, but the comprehensive status remains +0/8 until the final integrated candidate satisfies the complete gates. Do not +advertise the authoritative capability or claim program completion mid-stack. + +## Phase 4 — Run the complete proof matrix + +Use [`goalposts.md`](./goalposts.md) as the checklist. All thirteen production +journeys and every row in the mandatory +[`issue-to-journey matrix`](./related-open-work.md#mandatory-issue-to-journey-matrix) +are required. At minimum the final matrix must include: + +- real macOS, Linux floor, Windows, and WSL; +- local, daemon, direct SSH, nested SSH, paired runtime, and remote server; +- one Docker host with `MaxSessions=1` and two simultaneous Docker hosts; +- git worktree, folder, floating, drive, and UNC workspaces; +- old/new peers in both directions at every changed app/daemon, SSH relay, + paired-runtime, remote-server, and mobile/E2EE boundary; +- app, renderer, daemon, relay, and remote-runtime restart; +- crash cuts around durable operations and identity reset; +- unknown/timeout/disconnect/lost-response/concurrent-replacement cases; +- actual remote PID and process accounting; and +- A/B performance and memory results under the predeclared protocol, including + raw samples, confidence intervals, deterministic counters, and leak slopes. + +## Phase 5 — Delete, census, rebase, and review + +1. Trace every new production module from a real entrypoint. +2. Delete unreachable modules and fixtures in production trees. +3. Delete all superseded legacy writers and reconciliation state machines after + cutover. +4. Rebase onto the actual PR base. +5. Rerun all journeys and static/package checks on the rebased SHA. +6. Report production, tests, docs, CI/runner, generated, and vendored LOC + separately. +7. Minimise aggregate production source net LOC against the frozen pre-program + baseline, including all program-attributable carry-in and stack layers, and + justify every net addition as required by D1. +8. Run an independent repository review and the release-readiness checklist. +9. Resolve every P0–P2 finding. +10. Only then prepare the comprehensive PR and ask the user to approve any + deliberate design change. + +## Validation status at this handoff + +This was a documentation and read-only audit turn. No product test command was +rerun, so this handoff claims no fresh local product-test pass. + +Exact validation commands run for this handoff: + +```bash +git status --short --branch --untracked-files=all +git rev-parse HEAD origin/main +git rev-list --left-right --count HEAD...origin/main +git merge-base HEAD origin/main +git diff --numstat --no-renames "$(git merge-base HEAD origin/main)"..HEAD +gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +handoff_query_text="$(tr '\n' ' ' < docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql)" +gh api graphql -f query="$handoff_query_text" +pnpm exec oxfmt --check docs/reference/terminal-session-correctness-handoff-2026-08-07/*.md docs/reference/terminal-session-correctness-handoff-2026-08-07/*.graphql +git diff --check +``` + +Outcomes: + +- repository commands exited 0 and produced the branch/SHA/distance recorded in + `README.md`; +- the current-HEAD numstat reproduced `+309` production, `+914` tests/E2E, + `+146` docs, `+43` CI/config, and `+1,412` total net LOC; +- the reproducible GraphQL catalog returned all 65 referenced items and one + unresolved, non-outdated review thread on each of #13110 and #13111; +- all 30 linked issues appear in the mandatory issue-to-journey matrix and the + reusable GraphQL query; +- #13111 was open, non-draft, mergeable/clean, with 47 successful and 5 skipped + checks and no approval decision; and +- formatting and whitespace checks exited 0 after documentation formatting. + +Verified in this worktree: + +- repository branch, HEAD, merge base, local `origin/main`, branch distance, + and clean pre-documentation status; +- snapshot branch/commit presence and the 814-path preservation claim; +- current source implementation of the narrow #13111 behavior and gaps; +- current categorized #13111 diff census; +- original G0–G7, thirteen journeys, and PR proof contract; and +- live GitHub PR/issue state through batched GraphQL queries. + +Live GitHub verification at the time of writing showed: + +- #13111: open, non-draft, mergeable/clean, checks successful, no approval, one + unresolved major review thread; +- #13110: open, non-draft, checks successful, no approval, one unresolved major + review thread; +- #12264 and #12743: open competing STA-3077 implementations; +- #12474, #12477, and #12600: merged; and +- the other items in `related-open-work.md`: open unless explicitly labeled + merged or draft. + +These states are ephemeral and must be refreshed. + +Historical pause receipts from the 60,903-line construction snapshot remain +construction evidence only. They do not prove the current branch, G0–G7, or any +required journey. + +## Stop conditions + +Stop and return to the user before: + +- narrowing or replacing a named goalpost; +- accepting a correctness/performance tradeoff; +- deciding that remote orphan shells may be killed or leaked permanently; +- changing wire semantics without a compatibility design; +- shipping a positive-production-LOC comprehensive stack; +- merging, closing, commenting on, or force-updating external PRs; or +- deleting preserved user work. + +## Short takeover prompt + +> Continue the comprehensive terminal-session correctness program from +> [`README.md`](./README.md). Preserve the fixed status of **0/8 proven gates +> and 0/13 proven journeys** until every proof in +> [`goalposts.md`](./goalposts.md) passes on one rebased candidate. Reconcile the +> conflicting open work in [`related-open-work.md`](./related-open-work.md), +> build discriminating real SSH/daemon/paired/remote/cross-platform journeys, +> remove root causes and superseded reconciliation, minimise and justify every +> production-code addition per D1, and finish with no correctness or performance +> regression. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md new file mode 100644 index 00000000000..29083dfbb2a --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md @@ -0,0 +1,277 @@ +# Terminal session correctness — test design + +How we decide a test is worth trusting, what we built, and what we could not +prove. Read [`design-overview.md`](./design-overview.md) first. + +--- + +## 1. The problem with "the tests pass" + +A green test proves nothing on its own. It might pass because the code is +correct, or because the test never looked. + +```mermaid +flowchart TD + T["Test is green"] --> Q{"Would it go red if
we broke the code?"} + Q -->|"Yes"| G["Real evidence"] + Q -->|"No"| W["Worthless.
Green either way."] + Q -->|"Never checked"| U["Unknown.
Treat as worthless."] + + style G fill:#e6ffe6,stroke:#00aa00 + style W fill:#ffcccc,stroke:#cc0000 + style U fill:#fff4e6,stroke:#dd8800 +``` + +This is not hypothetical. **It happened to us twice:** + +- An end-to-end test was reported as proving the reconnect fix. It passed with + the fix **and** with the fix removed. Retracted. +- A guard was added, every test stayed green, and the guard had **no production + caller at all**. The capability existed; nothing used it. Tests passed the + whole time because they called the capability directly. + +--- + +## 2. The rule we adopted: the four-step proof + +Nothing counts as evidence until it has been through this loop. + +```mermaid +flowchart LR + S1["1. Run it
green"] --> S2["2. Delete the guard
from production code"] + S2 --> S3["3. Run it
MUST go red"] + S3 --> S4["4. Put the guard back
green again"] + S3 -.->|"stays green"| F["Test is worthless.
Say so. Don't ship it
as proof."] + + style S3 fill:#e6ffe6,stroke:#00aa00 + style F fill:#ffcccc,stroke:#cc0000 +``` + +Three rules that make this honest rather than theatre: + +**Delete a real guard, not the test.** The mutation goes in _production_ code. If +you tweak the test until it fails, you have proved nothing. + +**Prefer one clause at a time.** If a test has three claims, a good mutation +reddens exactly one and leaves the other two green. That proves the claims +_independently_ rather than as a bundle. + +**Watch it fail yourself.** A report saying "it went red" is not the same as +seeing it. Of the twelve oracles here, six were personally re-verified, and two +of those re-checks overturned the original claim. + +### The trap that nearly fooled us twice + +```mermaid +flowchart TD + M["Apply mutation"] --> C{"Did the mutation
actually land?"} + C -->|"No — silent no-op"| X["Test stays green.
You conclude 'no teeth'.
Wrong conclusion."] + C -->|"Yes"| R["Now the result
means something"] + + style X fill:#ffcccc,stroke:#cc0000 +``` + +Twice a search-and-replace matched the wrong line and changed nothing. The test +stayed green and looked like a failed oracle. **Always verify the mutation landed +before believing the result.** + +--- + +## 3. The three layers of test + +```mermaid +flowchart TD + subgraph L1["Unit — milliseconds"] + U1["Pure logic.
Does the rule compute
the right answer?"] + end + subgraph L2["Integration — seconds"] + I1["Real Electron app.
Real restarts.
Real shell processes."] + end + subgraph L3["End-to-end — minutes"] + E1["Real Docker container
running real OpenSSH.
Real network faults."] + end + L1 --> L2 --> L3 + L1 -.->|"fast, narrow"| N1[" "] + L3 -.->|"slow, real"| N2[" "] + style N1 fill:none,stroke:none + style N2 fill:none,stroke:none +``` + +The rule of thumb: **assert the strongest thing you can actually observe.** + +--- + +## 4. Why "same id" is not good enough + +The most valuable lesson from this work. + +A restart test naturally asks: _is it the same terminal?_ The easy check is +whether the ids match. That check is **too weak**, and we can prove it. + +```mermaid +flowchart TD + B["Break the quit path so it
destroys shells instead of
detaching from them"] --> R["Restart the app"] + R --> C1["tab id: same ✅"] + R --> C2["pane id: same ✅"] + R --> C3["pty id: same ✅"] + R --> C4["OS process: DIFFERENT ❌
pid 13756 → 8852"] + C3 --> W["A weak test passes here.
Your shell is gone."] + C4 --> S["Our test fails here."] + + style W fill:#ffcccc,stroke:#cc0000 + style S fill:#e6ffe6,stroke:#00aa00 +``` + +So the test asks the shell to report its **own process id** through the real +input path, then reads that process's **kernel start time**. Same id _and_ same +start time means the same process really survived — a recycled pid cannot pass. + +```mermaid +sequenceDiagram + participant T as Test + participant A as Orca + participant S as Shell + participant K as Kernel + T->>A: type "echo MARKER=$$" + A->>S: (through the real write path) + S-->>T: MARKER=13756 + T->>K: start time of 13756? + K-->>T: 02:01:29.337 + Note over T: restart the app + T->>S: ask again + S-->>T: MARKER=13756 + T->>K: start time of 13756? + K-->>T: 02:01:29.337 ✅ same process +``` + +--- + +## 5. What is covered + +### Journey 1 — a terminal survives restart · **PROVEN** + +Two claims: the same pane, binding and **OS process** survive a reload and an app +restart; and a stale operation is refused. + +| Platform | Result | Mutation A
(destructive quit) | Mutation B
(remove stale-write guard) | +| ------------------ | ------ | --------------------------------- | ----------------------------------------- | +| macOS | 2 pass | red | red | +| Linux, native | 2 pass | red | red | +| Windows 11, native | 2 pass | **red — only claim 1** | **red — only claim 2** | + +The two mutations hit **different claims**, which is what proves them separately. + +### Journey 2 — daemon and WSL · **PROVEN** + +| Environment | Result | Mutation A
(liveness can't say "unknown") | Mutation B
(widen owner fallback) | +| --------------- | ------ | --------------------------------------------- | ------------------------------------- | +| macOS | 3 pass | red — claim 1 only | red — claim 3 only | +| Linux, native | 3 pass | red — claim 1 only | red — claim 3 only | +| WSL2 on Windows | 3 pass | red — claim 1 only | — | + +> **A subtlety worth knowing.** These tests run in order, so when the first one +> fails the others report "did not run" — that is a _skip_, not a pass. To prove +> the mutation really only hit claim 1, we re-ran claims 2 and 3 **alone, with +> the mutation still applied**, and watched them stay green. + +### Other oracles that bite + +| Oracle | Mutation that reddens it | +| ---------------------------- | ------------------------------------------------ | +| Reconnect grafts no pane | Remove "bind, never create" | +| Superseded-keystroke fence | Remove the guard from the handlers | +| Guard is actually wired up | Remove the call sites, keep the function | +| Workspace ids don't collide | Restore the suffix-stripping comparison | +| Wire compatibility | Restore the wrong "expired" message | +| `rm -rf` hazard stays closed | Disable the quarantine → `cho hi; rm -rf x` runs | + +That last one is why we did **not** delete the quarantine module. + +--- + +## 6. What we could not prove, and why + +This is the honest half. + +```mermaid +flowchart TD + P["Property we want
to prove"] --> Q{"Is there a shipped
guard protecting it?"} + Q -->|"Yes"| M["Remove it → test reddens.
Provable"] + Q -->|"No — it's true
by construction"| C["Nothing to remove.
Not provable by mutation"] + Q -->|"No code exists
at all"| N["Nothing to test.
Not writable"] + + style M fill:#e6ffe6,stroke:#00aa00 + style C fill:#fff4e6,stroke:#dd8800 + style N fill:#ffcccc,stroke:#cc0000 +``` + +| Journey | Why not proven | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| Two hosts don't interfere | True **by construction** — each host has its own channel, so there is no shared thing to break. Four attempts to redden it failed, correctly. | +| Lazy host discovery | One claim has no mutation that hits it alone; removing the real guard breaks setup before the claim is reached. | +| `MaxSessions=1` reconnect | Two claims redden. A third survived **four** different guard removals — nothing shipped protects it. | +| Mixed versions, live | The new message **never crosses a version boundary** in production. Our in-process tests exercise a route the real app doesn't take. Withdrawn. | +| Performance | 1 of 10 required dimensions measured. The keystroke guard costs ~14ns, but measured outside the real app. | +| Three others | The features they test **do not exist** in the code. No test can be written. | + +**"True by construction" is a good outcome, not a failure** — it means the +property holds because of how the code is shaped, not because a guard is +watching. It just can't be demonstrated by breaking something. + +--- + +## 7. Environment traps that can fake a result + +Real problems we hit, worth knowing before trusting any run. + +```mermaid +flowchart TD + subgraph T1["Shared temp directory"] + A1["Test harness writes a pointer
to a machine-wide temp path"] --> A2["Two runs at once
overwrite each other"] + A2 --> A3["Can fake a red
and hide a real one"] + end + subgraph T2["Stale build"] + B1["e2e runs a built app"] --> B2["Edit source, forget rebuild"] + B2 --> B3["You tested the old code"] + end + style A3 fill:#ffcccc,stroke:#cc0000 + style B3 fill:#ffcccc,stroke:#cc0000 +``` + +So every run isolates its temp directory, and we check build timestamps against +source edits before believing a result. + +Two more, found by running on machines we don't develop on: + +- A test failed on **every** Linux run. Not a product bug: it read a value once, + immediately after a reload, where its sibling helper polls for 15 seconds. +- A test could not run on **Windows at all** — it used `echo $$` and `ps`, which + don't exist there. The Windows claim wasn't unproven, it was _unprovable_. + +Both were bugs in tests we wrote, found only by running on the real platform. + +--- + +## 8. Scoreboard + +```mermaid +pie showData + title Journeys + "Proven with discrimination" : 2 + "Oracle exists, cannot discriminate" : 5 + "No code exists to test" : 3 + "Not yet attempted" : 3 +``` + +| Measure | Value | +| ------------------------------- | ----------- | +| Journeys proven | **2 of 13** | +| Gates proven | **0 of 8** | +| Oracles that bite | 12 | +| …personally re-verified | 6 | +| Claims retracted after checking | 2 | +| Production code added | +214 lines | + +The retraction count is the number to look at. Two claims that looked green were +withdrawn after checking. That is the process working — and the reason to trust +the two journeys that survived it. diff --git a/package.json b/package.json index 1c2c1deb2df..2f2954020df 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,9 @@ "test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs", "test:e2e:ssh-docker-watcher-isolation": "node config/scripts/run-ssh-docker-watcher-isolation-e2e.mjs", "test:e2e:ssh-docker-terminal-parking": "node config/scripts/run-ssh-docker-terminal-parking-e2e.mjs", + "test:e2e:ssh-docker-reconnect-pane-cardinality": "node config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs", + "test:e2e:ssh-docker-maxsessions-identity": "node config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs", + "test:e2e:ssh-docker-two-hosts-isolation": "node config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs", "test:e2e:nested-runtime-ssh": "node config/scripts/run-nested-runtime-ssh-e2e.mjs", "test:e2e:source-control-scale": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/source-control-large-file-count.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "win-update-e2e": "node tests/tools/win-update-e2e/run.mjs", @@ -124,6 +127,7 @@ "bench:cold-park-resource": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs", "bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs", "test:e2e:remote-bulk-open-freeze": "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "test:e2e:daemon-session-liveness": "pnpm run ensure:electron-runtime && pnpm exec playwright test --config tests/playwright.daemon-liveness.local.config.ts", "test:e2e:ssh-docker-bulk-open-freeze": "node config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs", "repro:live-remote-bulk-open-freeze": "node config/scripts/live-remote-bulk-open-freeze-repro.mjs", "repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs" diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index 1e41e9de9ee..1fe5bc9770e 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -1911,6 +1911,20 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(adapter.hasPty(id)).toBe(true) }) + // Why: off-socket the cache can only be missing exits it never received, so a + // miss is ignorance. Answering false there authorizes a respawn over a live shell. + it('answers unknown, not absent, for an uncached id while disconnected', async () => { + const { id } = await adapter.spawn({ cols: 80, rows: 24 }) + const unknownId = 'repo::/repo/unknown@@deadbeef' + expect(adapter.hasPty(unknownId)).toBe(false) + + await server.shutdown() + await waitFor(() => !(adapter as unknown as { client: DaemonClient }).client.isConnected()) + + expect(adapter.hasPty(unknownId)).toBe(null) + expect(adapter.hasPty(id)).toBe(true) + }) + it('returns active sessions', async () => { await adapter.spawn({ cols: 80, diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index b2ccf55f6d7..49bc8b5a34b 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -940,8 +940,10 @@ export class DaemonPtyAdapter implements IPtyProvider { return providerSequence ? { providerSequence } : undefined } - hasPty(id: string): boolean { - return this.activeSessionIds.has(id) + hasPty(id: string): boolean | null { + // Why null off-socket: the cache only tracks exits it received, so a miss while + // disconnected (or before the first listSessions) is ignorance, not absence. + return this.activeSessionIds.has(id) ? true : this.client.isConnected() ? false : null } async probePtyLiveness(id: string): Promise { diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index 41cc8016169..7ddc240cc74 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -529,6 +529,33 @@ describe('DaemonPtyRouter', () => { expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session') }) + // Why: an unreachable generation cannot certify that another generation's session + // is gone — collapsing its null to false hands a respawn a fabricated death. + it('answers unknown when any adapter cannot see the session', () => { + const current = createAdapter('current') + const legacy = createAdapter('legacy') + legacy.hasPty = vi.fn(() => null) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + expect(router.hasPty('unmapped-session')).toBe(null) + }) + + it('does not report terminal_gone for inspection when an adapter cannot answer', async () => { + const current = createAdapter('current') + const legacy = createAdapter('legacy') + legacy.hasPty = vi.fn(() => null) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.inspectProcess('unmapped-session')).resolves.toEqual({ + foregroundProcess: null, + hasChildProcesses: false + }) + + // Unproven routes must not be memoized as an owner. + const unanimous = new DaemonPtyRouter({ current: createAdapter('c2'), legacy: [] }) + await expect(unanimous.inspectProcess('unmapped-session')).rejects.toThrow('terminal_gone') + }) + it('discovers an unmapped live session from one coalesced inventory', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy', ['surviving-session']) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 5104268cbef..a3962e511be 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -77,12 +77,15 @@ export class DaemonPtyRouter implements IPtyProvider { return await this.adapterFor(id).attach(id) } - hasPty(id: string): boolean { + hasPty(id: string): boolean | null { const routed = this.sessionAdapters.get(id) if (routed) { return routed.hasPty(id) } - return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id)) + // Why: one adapter's ignorance is not another's absence proof — only a unanimous + // proven-absence across every generation may read as false. + const answers = this.allAdapters().map((adapter) => adapter.hasPty(id)) + return answers.includes(true) ? true : answers.includes(null) ? null : false } async probePtyLiveness(id: string): Promise { @@ -314,14 +317,19 @@ export class DaemonPtyRouter implements IPtyProvider { } private adapterForInspection(sessionId: string): DaemonPtyAdapter { - const adapter = + const owner = this.sessionAdapters.get(sessionId) ?? - this.allAdapters().find((candidate) => candidate.hasPty(sessionId)) - if (!adapter) { + this.allAdapters().find((candidate) => candidate.hasPty(sessionId) === true) + if (owner) { + this.sessionAdapters.set(sessionId, owner) + return owner + } + // Why: only unanimous proven absence is gone; an adapter that cannot answer routes + // to current without memoizing a route we never established. + if (this.hasPty(sessionId) === false) { throw new Error('terminal_gone') } - this.sessionAdapters.set(sessionId, adapter) - return adapter + return this.current } private allAdapters(): DaemonPtyAdapter[] { diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index 6b473537e9f..1982c4add1b 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -471,6 +471,22 @@ describe('DegradedDaemonPtyProvider', () => { }) }) + // Why: while degraded, a provider that cannot answer must not let inspection + // manufacture terminal_gone — that verdict retires a pane that may still be live. + it('answers unknown, and refuses terminal_gone, when no provider can answer', async () => { + const current = createDaemonAdapter('daemon') + const fallback = createProvider('fallback') + current.hasPty = vi.fn(() => null) + fallback.hasPty = vi.fn(() => null) + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + expect(provider.hasPty('unmapped-session')).toBe(null) + await expect(provider.inspectProcess('unmapped-session')).resolves.toEqual({ + foregroundProcess: null, + hasChildProcesses: false + }) + }) + it('caches a provider discovered by hasPty before routing later operations', () => { const current = createDaemonAdapter('daemon', ['daemon-session']) const fallback = createProvider('fallback') @@ -662,3 +678,22 @@ describe('DegradedDaemonPtyProvider', () => { expect(fallback.listProcesses).toHaveBeenCalledTimes(3) }) }) + +// A memoized route outlives the session it was established for: listProcesses +// drops ids missing from an authoritative inventory without an exit fanout. So a +// mapped owner that cannot answer must stay unknown — coercing it to a liveness +// proof is worse than the absence it replaced, because callers skip the real probe. +it('keeps a mapped owner that cannot answer unknown, and still probes', async () => { + const current = createDaemonAdapter('current', ['s1']) + const fallback = createProvider('fallback') + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + expect(provider.hasPty('s1')).toBe(true) + + current.hasPty = vi.fn(() => null) + expect(provider.hasPty('s1')).toBeNull() + + current.probePtyLiveness = vi.fn(async () => null) + await provider.probePtyLiveness('s1') + expect(current.probePtyLiveness).toHaveBeenCalled() +}) diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 545f66ca892..e5b67478812 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -89,14 +89,25 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { attach = (id: string): ReturnType => attachDaemonOwnedSession(this.providerFor(id), this.fallback, id) - hasPty(id: string): boolean { + hasPty(id: string): boolean | null { const mapped = this.sessionProviders.get(id) - return mapped ? (mapped.hasPty?.(id) ?? true) : this.findProviderForExistingSession(id) !== null + if (mapped) { + // Why not `?? true`: a route outlives the session it was memoized for, so an + // owner that cannot answer must stay unknown rather than become a liveness proof. + return mapped.hasPty ? mapped.hasPty(id) : true + } + if (this.findProviderForExistingSession(id)) { + return true + } + // Why: one provider that cannot answer makes absence unknown; only unanimous proof is false. + return this.allProviders().every((provider) => provider.hasPty?.(id) === false) ? false : null } async probePtyLiveness(id: string): Promise { const mapped = this.sessionProviders.get(id) - if (mapped && (mapped.hasPty?.(id) ?? true)) { + // Why only an explicit true short-circuits: an owner that cannot answer must + // fall through to the real probe instead of skipping it on a fabricated proof. + if (mapped && (mapped.hasPty ? mapped.hasPty(id) === true : true)) { return true } return await this.ownerRecovery.probe(id) @@ -180,9 +191,9 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { return this.providerFor(id).getForegroundProcess(id) } inspectProcess(id: string) { - return this.hasPty(id) - ? inspectPtyProviderProcess(this.providerFor(id), id) - : Promise.reject(new Error('terminal_gone')) + return this.hasPty(id) === false + ? Promise.reject(new Error('terminal_gone')) + : inspectPtyProviderProcess(this.providerFor(id), id) } async confirmForegroundProcess(id: string): Promise { return this.providerFor(id).confirmForegroundProcess?.(id) ?? null diff --git a/src/main/ipc/ssh-pty-output-intake-test-harness.ts b/src/main/ipc/__tests__/ssh-pty-output-intake-test-harness.ts similarity index 97% rename from src/main/ipc/ssh-pty-output-intake-test-harness.ts rename to src/main/ipc/__tests__/ssh-pty-output-intake-test-harness.ts index 9209b897284..c3fe0b26ac9 100644 --- a/src/main/ipc/ssh-pty-output-intake-test-harness.ts +++ b/src/main/ipc/__tests__/ssh-pty-output-intake-test-harness.ts @@ -3,7 +3,7 @@ import { SshPtyOutputIntake, type SshPtyOutputDataEvent, type SshPtyOutputIntakeDependencies -} from './ssh-pty-output-intake' +} from '../ssh-pty-output-intake' export function sshPtyOutputEvent( overrides: Partial = {} diff --git a/src/main/ipc/pty-superseded-operation-fence.test.ts b/src/main/ipc/pty-superseded-operation-fence.test.ts new file mode 100644 index 00000000000..f357954bc26 --- /dev/null +++ b/src/main/ipc/pty-superseded-operation-fence.test.ts @@ -0,0 +1,118 @@ +/** + * A mutating operation must not reach a PTY that has been superseded for its + * pane. Main keeps `ptyPaneKey` and `paneKeyPtyId` in lock-step, so their + * disagreement is proof the caller's id is stale — not an inference from + * absence, which is why an id with no recorded pane stays permitted. + * + * The fence lives in the main process because the renderer queues input: a + * keystroke buffered before a reattach would otherwise land on the successor. + */ +import { describe, expect, it } from 'vitest' + +/** Mirrors the maps and predicate in pty.ts. Kept structural so the oracle + * tests the RULE, not one implementation of it. */ +function makeFence() { + const ptyPaneKey = new Map() + const paneKeyPtyId = new Map() + const bind = (paneKey: string, ptyId: string): void => { + ptyPaneKey.set(ptyId, paneKey) + paneKeyPtyId.set(paneKey, ptyId) + } + const isSuperseded = (ptyId: string): boolean => { + const paneKey = ptyPaneKey.get(ptyId) + if (paneKey === undefined) { + return false + } + const currentPtyId = paneKeyPtyId.get(paneKey) + return currentPtyId !== undefined && currentPtyId !== ptyId + } + return { bind, isSuperseded, ptyPaneKey, paneKeyPtyId } +} + +const PANE = 'tab-1:3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + +describe('superseded PTY operation fence', () => { + it('refuses an operation whose pane has since bound a different PTY', () => { + const fence = makeFence() + fence.bind(PANE, 'pty-old') + // The pane reattaches onto a fresh PTY; the renderer still holds pty-old. + fence.bind(PANE, 'pty-new') + + expect(fence.isSuperseded('pty-old')).toBe(true) + expect(fence.isSuperseded('pty-new')).toBe(false) + }) + + it('permits the current PTY for its own pane', () => { + const fence = makeFence() + fence.bind(PANE, 'pty-live') + + expect(fence.isSuperseded('pty-live')).toBe(false) + }) + + // Unknown is not stale. Orphan cleanup targets exactly these ids, so refusing + // them would break the operation that reclaims leaked shells. + it('permits a PTY that owns no pane', () => { + const fence = makeFence() + + expect(fence.isSuperseded('pty-orphan')).toBe(false) + }) + + it('permits a PTY whose pane record was dropped', () => { + const fence = makeFence() + fence.bind(PANE, 'pty-live') + fence.paneKeyPtyId.delete(PANE) + + expect(fence.isSuperseded('pty-live')).toBe(false) + }) + + it('keeps sibling panes independent', () => { + const fence = makeFence() + const other = 'tab-1:8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + fence.bind(PANE, 'pty-a') + fence.bind(other, 'pty-b') + fence.bind(PANE, 'pty-a2') + + expect(fence.isSuperseded('pty-a')).toBe(true) + expect(fence.isSuperseded('pty-b')).toBe(false) + }) +}) + +describe('the fence is wired into every mutating handler', () => { + it('combines supersession and incarnation in the write fence', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + const start = source.indexOf('const isCurrentPtyWrite') + + expect(start).toBeGreaterThan(0) + expect(source.slice(start, start + 300)).toContain('isSupersededPtyId') + expect(source.slice(start, start + 300)).toContain('ptyIncarnationById') + }) + + // The capability existed for months and was never called from these handlers. + // Pin the call site, not the capability — that is the failure this program hit. + it.each(['pty:write', 'pty:writeAccepted', 'pty:resize', 'pty:signal'])( + '%s consults the fence', + async (channel) => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + // Anchor on the registration, not the first mention of the name. + const start = source.search( + new RegExp(`ipcMain\\.(?:on|handle)\\(\\s*'${channel.replace(':', ':')}'`) + ) + expect(start, `${channel} handler not found`).toBeGreaterThan(0) + expect(source.slice(start, start + 700)).toContain( + channel.startsWith('pty:write') ? 'isCurrentPtyWrite' : 'isSupersededPtyId' + ) + } + ) + + // pty:kill is deliberately NOT fenced: a superseded PTY is orphaned, and + // reclaiming it is exactly what the orphan-cleanup callers ask for. + it('leaves pty:kill unfenced on purpose', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + const start = source.search(/ipcMain\.handle\(\s*'pty:kill'/) + expect(start).toBeGreaterThan(0) + expect(source.slice(start, start + 500)).not.toContain('isSupersededPtyId') + }) +}) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index d44ad8ef651..29d2feb10b9 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -245,6 +245,7 @@ import { getLocalPtyProvider, isCurrentPtyExit, restorePtyIncarnation, + bindPaneShell, type PrepareCodexSessionResume } from './pty' import { __resetPersistedWindowsPathCacheForTests } from '../pty/windows-environment-path' @@ -5289,7 +5290,10 @@ describe('registerPtyHandlers', () => { paneKey?: string tabId?: string }) => ({ - id: 'ssh-pty' + id: 'ssh-pty', + // The host attests the shell's identity at spawn; the lease has to carry it or the + // reconnect fence has nothing to compare and attaches by pty id alone. + incarnationId: 'inc-host-ssh-pty' }) ) const store = { @@ -5375,18 +5379,23 @@ describe('registerPtyHandlers', () => { worktreeId: 'wt-1', tabId: 'tab-1', leafId, + // Without this the field exists, loads and is read — and is never written, so the + // reconnect fence silently permits everything while every other clause stays green. + incarnationId: 'inc-host-ssh-pty', state: 'attached' }) ) - expect(store.persistPtyBinding).toHaveBeenCalledWith( - { - worktreeId: 'wt-1', - tabId: 'tab-1', - leafId, - ptyId: 'ssh-pty' - }, - 'ssh:ssh-1' - ) + // STA-3077 step P inverted this: SSH pane bindings no longer go to a + // per-target `ssh:` partition. One home (`local`) only, so the + // hostId argument is gone — pin its absence, not its old value. + expect(store.persistPtyBinding).toHaveBeenCalledWith({ + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId, + ptyId: 'ssh-pty', + incarnationId: 'inc-host-ssh-pty' + }) + expect(store.persistPtyBinding.mock.calls.at(-1)).toHaveLength(1) store.upsertSshRemotePtyLease.mockClear() store.persistPtyBinding.mockClear() @@ -6820,6 +6829,38 @@ describe('registerPtyHandlers', () => { expect(fallbackShutdown).not.toHaveBeenCalled() }) + // Why: before the swap lands the pre-swap local provider owns no daemon id, so its + // "not in my table" would answer an authoritative dead for every restored session. + it('waits for the desktop startup barrier before pty:hasPty answers for a daemon id', async () => { + const barrier = makeDeferred() + const awaitLocalPtyProviderStartup = vi.fn(() => barrier.promise) + registerPtyHandlers( + mainWindow as never, + undefined, + undefined, + undefined, + undefined, + undefined, + { + awaitLocalPtyProviderStartup + } + ) + + const pending = handlers.get('pty:hasPty')!(null, { id: 'daemon-session' }) as Promise< + boolean | null + > + await Promise.resolve() + expect(awaitLocalPtyProviderStartup).toHaveBeenCalledTimes(1) + + installObservableDaemonTestProvider() + ;(getLocalPtyProvider() as { hasPty?: (id: string) => boolean | null }).hasPty = vi.fn( + (id: string) => id === 'daemon-session' + ) + barrier.resolve() + + await expect(pending).resolves.toBe(true) + }) + it('waits for the desktop startup barrier before runtime local kills resolve the provider', async () => { const barrier = makeDeferred() const awaitLocalPtyProviderStartup = vi.fn(() => barrier.promise) @@ -7840,6 +7881,7 @@ describe('registerPtyHandlers', () => { id: opts.sessionId ?? 'daemon-pty' })), write, + hasPty: vi.fn(() => true), resize: vi.fn(args.resize ?? (() => {})), getAppliedSize: vi.fn(args.getAppliedSize ?? (async () => args.applied)), kill: vi.fn(), @@ -8064,6 +8106,107 @@ describe('registerPtyHandlers', () => { expect(write).not.toHaveBeenCalled() }) + it('does not forward queued host input after the pane rebinds', async () => { + const write = setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 } }) + let resolveClaim: (claimed: boolean) => void = () => {} + const claimRemoteDesktopHost = vi.fn( + () => + new Promise((resolve) => { + resolveClaim = resolve + }) + ) + const runtime = { + setPtyController: vi.fn(), + createPreAllocatedTerminalHandle: vi.fn(() => null), + registerPty: vi.fn(), + getDriver: vi.fn(() => ({ kind: 'idle' })), + claimRemoteDesktopHost, + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const spawn = await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24, env: {} }) + const id = (spawn as { id: string }).id + const leafId = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + const bindingStore = { + persistPtyBinding: vi.fn(() => true), + getWorkspaceSession: vi.fn(() => getDefaultWorkspaceSession()) + } + bindPaneShell({ + store: bindingStore, + worktreeId: 'repo-1:wt-1', + tabId: 'tab-1', + leafId, + ptyId: id + }) + const claim = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:claimViewport') + const writeEvent = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:write') + + claim?.[1](mainWindowIpcEvent, { id, cols: 125, rows: 48 }) + writeEvent?.[1](mainWindowIpcEvent, { id, data: 'queued' }) + const accepted = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id, + data: 'accepted' + }) + bindPaneShell({ + store: bindingStore, + worktreeId: 'repo-1:wt-1', + tabId: 'tab-1', + leafId, + ptyId: 'pty-successor' + }) + resolveClaim(true) + + await expect(accepted).resolves.toBe(false) + await Promise.resolve() + expect(write).not.toHaveBeenCalled() + clearProviderPtyState(id) + clearProviderPtyState('pty-successor') + }) + + it('does not forward queued host input after the PTY incarnation changes', async () => { + const write = setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 } }) + let resolveClaim: (claimed: boolean) => void = () => {} + const runtime = { + setPtyController: vi.fn(), + createPreAllocatedTerminalHandle: vi.fn(() => null), + registerPty: vi.fn(), + getDriver: vi.fn(() => ({ kind: 'idle' })), + claimRemoteDesktopHost: vi.fn( + () => + new Promise((resolve) => { + resolveClaim = resolve + }) + ), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const spawn = await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24, env: {} }) + const id = (spawn as { id: string }).id + restorePtyIncarnation(id, 'incarnation-old') + const claim = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:claimViewport') + const writeEvent = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:write') + + claim?.[1](mainWindowIpcEvent, { id, cols: 125, rows: 48 }) + writeEvent?.[1](mainWindowIpcEvent, { id, data: 'queued' }) + const accepted = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id, + data: 'accepted' + }) + restorePtyIncarnation(id, 'incarnation-new') + resolveClaim(true) + + await expect(accepted).resolves.toBe(false) + await Promise.resolve() + expect(write).not.toHaveBeenCalled() + clearProviderPtyState(id) + }) + it('does not populate the remote reclaim cache when only a phone drives', async () => { const resizeSpy = vi.fn() setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 }, resize: resizeSpy }) @@ -8665,12 +8808,17 @@ describe('registerPtyHandlers', () => { } }): Promise<{ id: string }> } + let emitExit: ((event: { exitCode: number; signal: number }) => void) | undefined + const kill = vi.fn(() => emitExit?.({ exitCode: 0, signal: 0 })) const proc = { onData: vi.fn(), - onExit: vi.fn(), + onExit: vi.fn((callback) => { + emitExit = callback + return makeDisposable() + }), write: vi.fn(), resize: vi.fn(), - kill: vi.fn(), + kill, process: 'zsh', pid: 12345 } @@ -8725,7 +8873,7 @@ describe('registerPtyHandlers', () => { expect(store.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ expectedSourceBinding }) ) - expect(proc.kill).toHaveBeenCalledOnce() + expect(kill).toHaveBeenCalledOnce() }) it('reports lower-owner commit before rejecting an early-exited runtime incarnation', async () => { @@ -9639,6 +9787,9 @@ describe('registerPtyHandlers', () => { ) expect(runtime.noteTerminalSpawnCommand).not.toHaveBeenCalled() expect(store.persistPtyBinding).toHaveBeenCalledOnce() + // STA-3077 step P inverted this: the reattach writer no longer passes a + // hostId slot at all (it used to pass `undefined` here for the local case), + // so the call is now strictly single-argument. expect(store.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ worktreeId, @@ -9650,9 +9801,10 @@ describe('registerPtyHandlers', () => { ptyId: 'pty-persisted-owner', incarnationId: 'inc-stale-owner' } - }), - undefined + }) ) + expect(store.persistPtyBinding.mock.calls[0]!).toHaveLength(1) + expect(getPtyIdForPaneKey(paneKey)).toBe('pty-persisted-owner') expect( mainWindow.webContents.send.mock.calls.filter(([channel]) => channel === 'pty:spawned') ).toHaveLength(1) @@ -10351,9 +10503,139 @@ describe('registerPtyHandlers', () => { expect(runtime.onPtyExit).toHaveBeenCalledWith('pty-already-retired-owner', 0, undefined) }) - it('retires a dead owner from the exact SSH host session before fresh recovery', async () => { + // Sibling of the clause above, and the reason that one is driven by a proof. Retiring an owner + // authorizes a replacement carrying the pane's agent resume payload, so a bare not-found must + // not do it: a replaced relay answers exactly that for shells its predecessor still runs. + it('does not retire an SSH owner, nor respawn it, on a bare not-found', async () => { + const connectionId = 'ssh-unproven-stable-pane' + const tabId = 'tab-unproven-ssh-owner' + const leafId = '35353535-3535-4535-8535-353535353535' + const paneKey = makePaneKey(tabId, leafId) + const worktreeId = 'repo-ssh::/remote/unproven-stable-pane' + const livePtyId = `ssh:${connectionId}@@live-relay-pty` + const freshPtyId = `ssh:${connectionId}@@fresh-relay-pty` + const remoteSpawn = vi.fn(async (options: { attachOnly?: boolean; command?: string }) => { + if (options.attachOnly) { + // The reattach mints this only after verifying the relay's observed exit names this shell. + // A bare not-found reaches here too and must NOT retire the owner — pinned by the sibling + // clause below, because retirement authorizes a replacement carrying the resume payload. + throw new Error('PTY "live-relay-pty" not found') + } + return { id: freshPtyId, incarnationId: 'inc-fresh-ssh-owner' } + }) + registerSshPtyProvider(connectionId, { + spawn: remoteSpawn, + write: vi.fn(), + resize: vi.fn(), + shutdown: vi.fn(), + sendSignal: vi.fn(), + getCwd: vi.fn(), + getInitialCwd: vi.fn(), + clearBuffer: vi.fn(), + acknowledgeDataEvent: vi.fn(), + onData: vi.fn(() => () => {}), + onReplay: vi.fn(() => () => {}), + onExit: vi.fn(() => () => {}), + listProcesses: vi.fn(), + hasChildProcesses: vi.fn(), + getForegroundProcess: vi.fn(), + serialize: vi.fn(), + revive: vi.fn(), + getDefaultShell: vi.fn(), + getProfiles: vi.fn() + } as never) + let session = { + tabsByWorktree: { + [worktreeId]: [{ id: tabId, worktreeId, ptyId: livePtyId }] + }, + terminalLayoutsByTabId: { + [tabId]: { + root: { type: 'leaf' as const, leafId }, + activeLeafId: leafId, + expandedLeafId: null, + ptyIdsByLeafId: { [leafId]: livePtyId } + } + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: 'inc-live-ssh-owner' } + } + const store = { + // STA-3077 step P inverted these: the reader/retirer used to target the + // per-target `ssh:` partition while the renderer published + // pane membership to `local` — two homes, so supersession no-opped. Both + // sides now address the single default (local) partition, i.e. no hostId. + getWorkspaceSession: vi.fn((requestedHostId?: string) => { + expect(requestedHostId).toBeUndefined() + return session + }), + setWorkspaceSession: vi.fn((next, requestedHostId?: string) => { + expect(requestedHostId).toBeUndefined() + session = next + }), + flushOrThrow: vi.fn(), + persistPtyBinding: vi.fn(), + upsertSshRemotePtyLease: vi.fn(), + removeSshRemotePtyLease: vi.fn(), + markSshRemotePtyLease: vi.fn() + } + const runtime = { + setPtyController: vi.fn(), + resolveTerminalPane: vi.fn(() => { + throw new Error('terminal_not_found') + }), + createPreAllocatedTerminalHandle: vi.fn(() => 'term-fresh-ssh-owner'), + registerPreAllocatedHandleForPty: vi.fn(), + beginPtyRegistration: vi.fn(), + cancelPendingPtyRegistration: vi.fn(), + assertPtyRegistrationAllowed: vi.fn(), + registerPty: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + seedHeadlessTerminal: vi.fn(), + getDriver: vi.fn(() => ({ kind: 'host' })), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + + try { + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + await expect( + handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + cwd: '/remote/unproven-stable-pane', + command: 'codex resume a-session-that-may-still-be-running', + connectionId, + worktreeId, + tabId, + leafId, + env: { + ORCA_PANE_KEY: paneKey, + ORCA_TAB_ID: tabId, + ORCA_WORKTREE_ID: worktreeId + } + }) + ).rejects.toThrow(/not found/i) + + // Only the attach ran: no replacement carrying the resume payload. + expect(remoteSpawn).toHaveBeenCalledTimes(1) + expect(remoteSpawn.mock.calls[0]?.[0]).toMatchObject({ attachOnly: true }) + // The pane keeps its owner, so the shell stays reattachable. + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + } finally { + unregisterSshPtyProvider(connectionId) + } + }) + + it('retires a dead owner from the local session — the one pane-binding home — before fresh recovery', async () => { const connectionId = 'ssh-dead-stable-pane' - const hostId = `ssh:${connectionId}` const tabId = 'tab-dead-ssh-owner' const leafId = '34343434-3434-4434-8434-343434343434' const paneKey = makePaneKey(tabId, leafId) @@ -10362,7 +10644,8 @@ describe('registerPtyHandlers', () => { const freshPtyId = `ssh:${connectionId}@@fresh-relay-pty` const remoteSpawn = vi.fn(async (options: { attachOnly?: boolean; command?: string }) => { if (options.attachOnly) { - throw new Error('PTY "dead-relay-pty" not found') + // Proven exit: only this error authorizes retiring the owner (bare not-found must not). + throw new Error('SSH_SESSION_EXPIRED: dead-relay-pty') } return { id: freshPtyId, incarnationId: 'inc-fresh-ssh-owner' } }) @@ -10402,12 +10685,16 @@ describe('registerPtyHandlers', () => { terminalPtyIncarnationsByPaneKey: { [paneKey]: 'inc-dead-ssh-owner' } } const store = { + // STA-3077 step P inverted these: the reader/retirer used to target the + // per-target `ssh:` partition while the renderer published + // pane membership to `local` — two homes, so supersession no-opped. Both + // sides now address the single default (local) partition, i.e. no hostId. getWorkspaceSession: vi.fn((requestedHostId?: string) => { - expect(requestedHostId).toBe(hostId) + expect(requestedHostId).toBeUndefined() return session }), setWorkspaceSession: vi.fn((next, requestedHostId?: string) => { - expect(requestedHostId).toBe(hostId) + expect(requestedHostId).toBeUndefined() session = next }), flushOrThrow: vi.fn(), @@ -10470,16 +10757,25 @@ describe('registerPtyHandlers', () => { expect(remoteSpawn.mock.calls[1]?.[0]).toMatchObject({ command: 'codex resume exact-dead-ssh-provider-session' }) - expect(store.setWorkspaceSession).toHaveBeenCalledWith(expect.anything(), hostId) + // STA-3077 step P inverted these: the retirement write and the fresh + // binding write both land in the single default (local) partition now, so + // no hostId is passed. The dead owner must still actually be retired. + expect(store.setWorkspaceSession).toHaveBeenCalledOnce() + expect(store.setWorkspaceSession.mock.calls[0]!).toHaveLength(1) + expect( + store.setWorkspaceSession.mock.calls[0]![0].terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[ + leafId + ] + ).not.toBe(deadPtyId) expect(store.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ worktreeId, tabId, leafId, ptyId: freshPtyId - }), - hostId + }) ) + expect(store.persistPtyBinding.mock.calls.at(-1)!).toHaveLength(1) } finally { unregisterSshPtyProvider(connectionId) } @@ -10825,15 +11121,15 @@ describe('registerPtyHandlers', () => { state: 'attached' }) ) - expect(store.persistPtyBinding).toHaveBeenCalledWith( - { - worktreeId: 'wt-remote', - tabId: 'tab-remote', - leafId, - ptyId: 'ssh:ssh-1@@relay-pty' - }, - 'ssh:ssh-1' - ) + // STA-3077 step P inverted this: the headless binding writer no longer + // targets `ssh:`; one home (`local`) only, so no hostId argument. + expect(store.persistPtyBinding).toHaveBeenCalledWith({ + worktreeId: 'wt-remote', + tabId: 'tab-remote', + leafId, + ptyId: 'ssh:ssh-1@@relay-pty' + }) + expect(store.persistPtyBinding.mock.calls[0]!).toHaveLength(1) expect(store.persistPtyBinding.mock.invocationCallOrder[0]!).toBeLessThan( store.upsertSshRemotePtyLease.mock.invocationCallOrder[0]! ) @@ -10977,15 +11273,15 @@ describe('registerPtyHandlers', () => { persistHostSessionBinding: true }) - expect(store.persistPtyBinding).toHaveBeenCalledWith( - { - worktreeId: 'wt-remote', - tabId: 'tab-remote', - leafId, - ptyId: 'ssh:ssh-reattach-ok@@relay-pty' - }, - 'ssh:ssh-reattach-ok' - ) + // STA-3077 step P inverted this: the reattach binding refresh writes to the + // single default (local) partition, so the `ssh:` hostId is gone. + expect(store.persistPtyBinding).toHaveBeenCalledWith({ + worktreeId: 'wt-remote', + tabId: 'tab-remote', + leafId, + ptyId: 'ssh:ssh-reattach-ok@@relay-pty' + }) + expect(store.persistPtyBinding.mock.calls.at(-1)!).toHaveLength(1) expect(store.upsertSshRemotePtyLease).toHaveBeenCalledWith( expect.objectContaining({ targetId: 'ssh-reattach-ok', @@ -17660,6 +17956,31 @@ describe('registerPtyHandlers', () => { expect(mockProc.proc.write).toHaveBeenNthCalledWith(2, 'tail') }) + it('stops a chunked write when the PTY incarnation changes between chunks', async () => { + vi.useFakeTimers() + const mockProc = createMockProc() + spawnMock.mockReturnValue(mockProc.proc) + registerPtyHandlers(mainWindow as never) + const result = (await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24 + })) as { id: string } + restorePtyIncarnation(result.id, 'incarnation-old') + const text = ['x'.repeat(TERMINAL_INPUT_CHUNK_MAX_BYTES), 'tail'].join('') + + const writeResult = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id: result.id, + data: text + }) + expect(mockProc.proc.write).toHaveBeenCalledOnce() + restorePtyIncarnation(result.id, 'incarnation-new') + await vi.runAllTimersAsync() + + await expect(writeResult).resolves.toBe(false) + expect(mockProc.proc.write).toHaveBeenCalledOnce() + clearProviderPtyState(result.id) + }) + it('yields while validating accepted large acknowledged pty writes before provider writes', async () => { const mockProc = createMockProc() spawnMock.mockReturnValue(mockProc.proc) @@ -17684,6 +18005,30 @@ describe('registerPtyHandlers', () => { expect(mockProc.proc.write.mock.calls.map(([chunk]) => chunk).join('')).toBe(text) }) + it('drops a deferred write when the PTY incarnation changes during validation', async () => { + vi.useFakeTimers() + const mockProc = createMockProc() + spawnMock.mockReturnValue(mockProc.proc) + registerPtyHandlers(mainWindow as never) + const result = (await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24 + })) as { id: string } + restorePtyIncarnation(result.id, 'incarnation-old') + const text = 'é'.repeat(CLIPBOARD_TEXT_MEASURE_YIELD_CODE_UNITS + 1) + + const writeResult = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id: result.id, + data: text + }) + restorePtyIncarnation(result.id, 'incarnation-new') + await vi.runAllTimersAsync() + + await expect(writeResult).resolves.toBe(false) + expect(mockProc.proc.write).not.toHaveBeenCalled() + clearProviderPtyState(result.id) + }) + it('rejects oversized acknowledged pty writes before provider writes', async () => { const mockProc = createMockProc() spawnMock.mockReturnValue(mockProc.proc) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index c2877303561..4b18f55f7e5 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -15,8 +15,8 @@ export { getBashShellReadyRcfileContent } from '../providers/local-pty-shell-rea import type { OrcaRuntimeService } from '../runtime/orca-runtime' import type { PtyBindingSourceExpectation, Store } from '../persistence' import { retireTerminalSurfaceFromPersistence } from '../runtime/mobile-session-terminal-persistence-retirement' +import { findTerminalTabIdForLeaf } from '../runtime/workspace-session-terminal-membership-authority' import type { GlobalSettings, TuiAgent } from '../../shared/types' -import { toSshExecutionHostId } from '../../shared/execution-host' import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' import { terminalOutputBacklogCapChars } from '../../shared/terminal-scrollback-policy' import type { @@ -84,6 +84,7 @@ import { import type { StartupCommandDelivery } from '../../shared/codex-startup-delivery' import { SSH_SESSION_EXPIRED_ERROR, + isSshPtyExitedError, isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors' @@ -234,7 +235,7 @@ import { } from '../project-groups/folder-workspace-path-status' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { resolveLocalProjectRuntimeForWorktreeId } from '../local-project-runtime-resolution' -import { isPtyIncarnationId } from '../../shared/pty-incarnation' +import { isPtyIncarnationId, isRelayAttestedPtyIncarnationId } from '../../shared/pty-incarnation' import type { PtyListedSession } from '../../shared/pty-listed-session' // ─── Provider Registry ────────────────────────────────────────────── @@ -286,6 +287,26 @@ const ptyPaneKey = new Map() // Why: reverse of ptyPaneKey — callers with a paneKey from outside the PTY lifecycle (e.g. agent-hook status routing) need the ptyId; kept in lock-step via the same sites. const paneKeyPtyId = new Map() +/** + * True only when this PTY's own pane has since bound a different PTY. The two + * maps above are maintained in lock-step, so their disagreement is proof the id + * was superseded — a renderer that queued work before a reattach cannot land it + * on the successor. + * + * Deliberately false when no paneKey is recorded: an unowned or orphaned PTY is + * unknown, not stale, and unknown never authorizes refusing an explicit + * operation. That is also what keeps orphan cleanup working, since those ids + * have no pane by construction. + */ +function isSupersededPtyId(ptyId: string): boolean { + const paneKey = ptyPaneKey.get(ptyId) + if (paneKey === undefined) { + return false + } + const currentPtyId = paneKeyPtyId.get(paneKey) + return currentPtyId !== undefined && currentPtyId !== ptyId +} + const AGENT_HOOK_RUNTIME_ENV_KEYS = [ 'ORCA_AGENT_HOOK_PORT', 'ORCA_AGENT_HOOK_TOKEN', @@ -515,6 +536,68 @@ function rememberPaneKeyForPty(ptyId: string, paneKey: unknown): string | null { return normalizedPaneKey } +/** + * The one producer of a pane -> shell binding: durable record and fence maps together, keyed by the + * tab holding the leaf *now* (a stored tabId names the tab a moved pane left). Splitting these let + * the superseded-PTY fence sit inert on reattach, the path it was built for. + * + * `bound` is false when a durable pane refuses. A throw is unknown rather than a refusal, so it + * propagates — only the caller that created the shell should clean it up. + * + * `tabId` is the resolved tab, returned so a caller that also registers the pane with the runtime + * graph uses the SAME coordinate. Registering under the lease's frozen tabId while the record and + * the fence use the live one splits the pane across two tabs, and the graph half then ensures a + * mobile surface for the tab the pane left. + */ +/** + * The tab a leaf lives in *now*. Callers whose own `tabId` is fresh (spawn) must not use this — the + * persisted layout is the stale side inside the renderer's publish debounce. Callers holding a + * tabId frozen in a durable lease (reattach) must, because the pane may have been moved since. + * + * Resolved separately from `bindPaneShell` so a thrown durable write cannot lose the answer and + * leave the runtime graph registered under the tab the pane left. + */ +export function resolvePaneShellTabId( + store: Pick | undefined, + leafId: string +): string | undefined { + if (typeof store?.getWorkspaceSession !== 'function') { + return undefined + } + return findTerminalTabIdForLeaf(store.getWorkspaceSession(), leafId) +} + +export function bindPaneShell(args: { + store: Pick | undefined + worktreeId: string + tabId: string + leafId: string + ptyId: string + incarnationId?: string + startupCwd?: string + mayCreate?: boolean + expectedBinding?: { ptyId: string; incarnationId?: string } + expectedSourceBinding?: PtyBindingSourceExpectation +}): { bound: boolean; tabId: string } { + const tabId = args.tabId + const bound = args.store?.persistPtyBinding({ + worktreeId: args.worktreeId, + tabId, + leafId: args.leafId, + ptyId: args.ptyId, + ...(args.incarnationId ? { incarnationId: args.incarnationId } : {}), + ...(args.startupCwd ? { startupCwd: args.startupCwd } : {}), + ...(args.mayCreate === false ? { mayCreate: false } : {}), + ...(args.expectedBinding ? { expectedBinding: args.expectedBinding } : {}), + ...(args.expectedSourceBinding ? { expectedSourceBinding: args.expectedSourceBinding } : {}) + }) + if (bound === false) { + return { bound: false, tabId } + } + rememberPaneKeyForPty(args.ptyId, makePaneKey(tabId, args.leafId)) + return { bound: true, tabId } +} + function cleanupPendingPaneSerializersForSender(ownerWebContentsId: number): void { pendingPaneSerializerCleanupRegistered.delete(ownerWebContentsId) for (const [paneKey, pending] of pendingByPaneKey) { @@ -651,11 +734,13 @@ type StablePaneAdoption = { } | null const stablePaneAdoptionsByOwnerKey = new Map>() +// Pane bindings have one home: the local partition. An SSH pane is not partitioned out — the +// renderer publishes its membership to `local` and `mayCreate: false` is evaluated there — so +// selecting `ssh:` here read a partition no live writer maintains (STA-3077 step P). function resolvePersistedStablePaneOwner( store: Store | undefined, paneKey: string, - worktreeId: string, - connectionId: string | null | undefined + worktreeId: string ): Pick | null { if (!store || typeof store.getWorkspaceSession !== 'function') { return null @@ -664,9 +749,7 @@ function resolvePersistedStablePaneOwner( if (!parsed) { return null } - const session = store.getWorkspaceSession( - connectionId ? toSshExecutionHostId(connectionId) : undefined - ) + const session = store.getWorkspaceSession() const tab = session.tabsByWorktree?.[worktreeId]?.find( (candidate) => candidate.id === parsed.tabId && candidate.worktreeId === worktreeId ) @@ -706,7 +789,7 @@ function resolveStablePaneOwner( } } } - const persisted = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) + const persisted = resolvePersistedStablePaneOwner(store, paneKey, worktreeId) if (resolved?.ptyId && persisted && resolved.ptyId !== persisted.ptyId) { throw new Error('terminal_pane_owner_conflict') } @@ -742,15 +825,13 @@ function resolveStablePaneOwner( function retirePersistedStablePaneOwner( store: Store | undefined, owner: StablePaneOwner, - worktreeId: string, - connectionId: string | null | undefined + worktreeId: string ): boolean { if (!store) { return false } const paneKey = makePaneKey(owner.tabId, owner.leafId) - const hostId = connectionId ? toSshExecutionHostId(connectionId) : undefined - const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) + const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId) if (!current) { // Why: persistence already dropped this pane binding (an earlier stop retired it while the // runtime kept history), so there is nothing left to clear — that is a completed retirement, @@ -760,7 +841,7 @@ function retirePersistedStablePaneOwner( if (current.ptyId !== owner.ptyId || current.incarnationId !== owner.persistedIncarnationId) { return false } - const session = store.getWorkspaceSession(hostId) + const session = store.getWorkspaceSession() const retired = retireTerminalSurfaceFromPersistence(session, { worktreeId, parentTabId: owner.tabId, @@ -771,7 +852,7 @@ function retirePersistedStablePaneOwner( if (retired === session) { return false } - store.setWorkspaceSession(retired, hostId) + store.setWorkspaceSession(retired) store.flushOrThrow() return true } @@ -786,6 +867,8 @@ type StablePaneSpawnContext = { connectionId?: string | null resolveOwner?: () => StablePaneOwner | null onFreshSpawn?: (result: PtySpawnResult) => void + /** Create a shell for this pane instead of attaching the one it records; see the caller. */ + refuseAdoption?: boolean } function stablePanePersistenceFence( @@ -805,25 +888,22 @@ function persistAdmittedStablePaneBinding(args: { result: PtySpawnResult worktreeId: string | undefined startupCwd: string | undefined - connectionId: string | null | undefined }): boolean { const expectedBinding = stablePanePersistenceFence(args.owner) if (!args.store || !args.owner || !args.worktreeId || !expectedBinding) { return false } - const persisted = args.store.persistPtyBinding( - { - worktreeId: args.worktreeId, - tabId: args.owner.tabId, - leafId: args.owner.leafId, - ptyId: args.result.id, - ...(args.result.incarnationId ? { incarnationId: args.result.incarnationId } : {}), - ...(args.startupCwd ? { startupCwd: args.startupCwd } : {}), - expectedBinding - }, - args.connectionId ? toSshExecutionHostId(args.connectionId) : undefined - ) - if (persisted === false) { + const persisted = bindPaneShell({ + store: args.store, + worktreeId: args.worktreeId, + tabId: args.owner.tabId, + leafId: args.owner.leafId, + ptyId: args.result.id, + ...(args.result.incarnationId ? { incarnationId: args.result.incarnationId } : {}), + ...(args.startupCwd ? { startupCwd: args.startupCwd } : {}), + expectedBinding + }) + if (persisted.bound === false) { throw new Error('terminal_pane_owner_changed') } return true @@ -859,7 +939,7 @@ async function attachStablePaneOwner( if (isDaemonEndpointGoneError(error)) { throw new TerminalHostGoneError() } - if (!isPtyAlreadyGoneError(error)) { + if (!isPtyProvenGoneForReplacement(error, owner.ptyId)) { throw error } const ownerBeforeRetire = args.resolveOwner?.() @@ -875,10 +955,7 @@ async function attachStablePaneOwner( runtime?.onPtyExit(owner.ptyId, 0, owner.incarnationId) clearProviderPtyState(owner.ptyId) ptyOwnership.delete(owner.ptyId) - if ( - args.worktreeId && - !retirePersistedStablePaneOwner(args.store, owner, args.worktreeId, args.connectionId) - ) { + if (args.worktreeId && !retirePersistedStablePaneOwner(args.store, owner, args.worktreeId)) { throw new Error('terminal_pane_owner_changed') } if (args.resolveOwner?.()) { @@ -901,7 +978,11 @@ async function attachStablePaneOwner( async function spawnForStablePane( args: StablePaneSpawnContext ): Promise<{ result: PtySpawnResult; owner: StablePaneOwner | null }> { - if (args.owner) { + // The one place an owner becomes `sessionId` for the provider, which is what makes an attach an + // attach. Refusing adoption has to be honoured HERE: gating only where the owner is resolved + // leaves every other resolution free to reach this line, which is exactly how the unreachable + // pane's "start a new terminal" kept attaching the shell it could not reach. + if (args.owner && !args.refuseAdoption) { const attached = await attachStablePaneOwner({ ...args, owner: args.owner }) if (attached) { return attached @@ -1028,9 +1109,29 @@ function normalizeNodePtySpawnError(err: unknown): Error { return new Error(hintedMessage) } +/** + * Retiring a pane's owner authorizes a replacement that carries its agent resume payload, so it + * needs proof — unlike a shutdown, where "not found" is simply the outcome we asked for. A bare + * not-found proves an exit only from a provider that owns its ptys; a replacement relay answers the + * same for shells its predecessor is still running. + */ +function isPtyProvenGoneForReplacement(err: unknown, ptyId: string): boolean { + if (!parseAppSshPtyId(ptyId)) { + return isPtyAlreadyGoneError(err) + } + // For SSH, the two proving answers are the relay's own observed exit and the expiry the reattach + // mints only after verifying that proof names this shell. A bare not-found is neither. + const message = err instanceof Error ? err.message : String(err) + return isSshPtyExitedError(err) || message.includes(SSH_SESSION_EXPIRED_ERROR) +} + function isPtyAlreadyGoneError(err: unknown): boolean { const message = err instanceof Error ? err.message : String(err) - return isSshPtyNotFoundError(err) || /Session not found/i.test(message) + // A proven exit is the strongest form of "already gone"; it only reads differently because the + // relay now says what it observed instead of reporting an unknown id. + return ( + isSshPtyNotFoundError(err) || isSshPtyExitedError(err) || /Session not found/i.test(message) + ) } function delay(ms: number): Promise { @@ -5132,8 +5233,7 @@ export function registerPtyHandlers( owner: stablePaneOwner, result, worktreeId: hostSessionBinding?.worktreeId, - startupCwd: cwd, - connectionId: args.connectionId + startupCwd: cwd }) } catch (error) { if (error instanceof Error && error.message === 'terminal_pane_owner_changed') { @@ -5201,6 +5301,12 @@ export function registerPtyHandlers( ...(typeof args.leafId === 'string' && isTerminalLeafId(args.leafId) ? { leafId: args.leafId } : {}), + // The shell this lease names, as the host just attested it. Without it the record says + // only "pty-N", and a replaced relay reissues that from 1 — so reconnect would have + // nothing to compare and would attach whatever now answers to the id. + ...(isRelayAttestedPtyIncarnationId(result.incarnationId) + ? { incarnationId: result.incarnationId } + : {}), state: 'attached', lastAttachedAt: Date.now() }) @@ -5224,7 +5330,8 @@ export function registerPtyHandlers( }) if (hostSessionBinding && !stablePaneBindingPersisted) { try { - const binding = { + const bound = bindPaneShell({ + store: hostSessionBinding.store, worktreeId: hostSessionBinding.worktreeId, tabId: hostSessionBinding.tabId, leafId: hostSessionBinding.leafId, @@ -5234,14 +5341,8 @@ export function registerPtyHandlers( ...(hostSessionBinding.expectedSourceBinding ? { expectedSourceBinding: hostSessionBinding.expectedSourceBinding } : {}) - } - const persisted = args.connectionId - ? hostSessionBinding.store.persistPtyBinding( - binding, - toSshExecutionHostId(args.connectionId) - ) - : hostSessionBinding.store.persistPtyBinding(binding) - if (persisted === false) { + }) + if (bound.bound === false) { throw new Error('terminal_split_source_not_found') } } catch (err) { @@ -5845,6 +5946,8 @@ export function registerPtyHandlers( cwd?: string // Why: fresh local spawns opt into recovering a saved cwd whose dir was deleted (#7239); reattach/remote need exact cwd, so the flag alone isn't sufficient. cwdFallback?: 'worktree' + /** Create a shell for this pane rather than adopting the one it records. */ + createFreshShellForUnreachablePane?: boolean env?: Record envToDelete?: string[] command?: string @@ -5944,8 +6047,14 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return { ...(await existingPaneSpawn.promise), isReattach: true } } + // Why the caller may refuse adoption: the unreachable-pane card offers "Start a new terminal" + // for a pane whose recorded shell cannot be reached. Resolving an owner here makes the action + // attach that shell first — and when it is unreachable the attach fails, the action creates + // nothing, and the card comes back. The button is dead in the one state it is offered in. + // Skipping the resolve is what makes it a creation: nothing is killed, and the old shell is + // left alive and unbound for the cleanup surface. const earlyStablePaneOwner = - earlyPaneKey && args.worktreeId + earlyPaneKey && args.worktreeId && args.createFreshShellForUnreachablePane !== true ? resolveStablePaneOwner(runtime, store, earlyPaneKey, args.worktreeId, args.connectionId) : null const earlyWorktreeId = args.worktreeId @@ -6555,6 +6664,7 @@ export function registerPtyHandlers( provider, spawnOptions, owner: stablePaneOwnerCandidate, + refuseAdoption: args.createFreshShellForUnreachablePane === true, worktreeId: args.worktreeId, connectionId: args.connectionId, resolveOwner: () => @@ -6694,8 +6804,7 @@ export function registerPtyHandlers( owner: stablePaneOwner, result, worktreeId: args.worktreeId, - startupCwd: cwd, - connectionId: args.connectionId + startupCwd: cwd }) } catch (error) { if (error instanceof Error && error.message === 'terminal_pane_owner_changed') { @@ -6748,6 +6857,11 @@ export function registerPtyHandlers( ...(typeof args.worktreeId === 'string' ? { worktreeId: args.worktreeId } : {}), ...(typeof args.tabId === 'string' ? { tabId: args.tabId } : {}), ...(validatedLeafId ? { leafId: validatedLeafId } : {}), + // See the sibling writer: a lease that names only "pty-N" cannot survive a relay that + // reissues ids from 1. + ...(isRelayAttestedPtyIncarnationId(result.incarnationId) + ? { incarnationId: result.incarnationId } + : {}), state: 'attached', lastAttachedAt: Date.now() }) @@ -6768,19 +6882,15 @@ export function registerPtyHandlers( !stablePaneBindingPersisted ) { try { - const binding = { + bindPaneShell({ + store, worktreeId: args.worktreeId, tabId: args.tabId, leafId: validatedLeafId, ptyId: result.id, ...(result.incarnationId ? { incarnationId: result.incarnationId } : {}), ...(cwd ? { startupCwd: cwd } : {}) - } - if (args.connectionId) { - store.persistPtyBinding(binding, toSshExecutionHostId(args.connectionId)) - } else { - store.persistPtyBinding(binding) - } + }) } catch (err) { console.error('[pty] failed to persist PTY binding after spawn:', err) if (!result.isReattach) { @@ -7049,11 +7159,20 @@ export function registerPtyHandlers( mainWindow.webContents.send('pty:writeUnavailable', { id }) } + type PtyWriteFence = { incarnationId: string | undefined } + + const isCurrentPtyWrite = (id: string, fence: PtyWriteFence): boolean => + !isSupersededPtyId(id) && ptyIncarnationById.get(id) === fence.incarnationId + const writePtyProviderInputWithinLimit = ( provider: IPtyProvider, id: string, - data: string + data: string, + fence: PtyWriteFence ): boolean | Promise => { + if (!isCurrentPtyWrite(id, fence)) { + return false + } const chunks = iterateTerminalInputChunks(data) const first = chunks.next() if (first.done) { @@ -7065,21 +7184,24 @@ export function registerPtyHandlers( provider.write(id, first.value) return true } - return writePtyProviderInputChunks(provider, id, chunks, first.value, second.value) + return writePtyProviderInputChunks(provider, id, chunks, first.value, second.value, fence) } const writePtyProviderInput = ( provider: IPtyProvider, id: string, - data: string + data: string, + fence: PtyWriteFence ): boolean | Promise => { try { const tooLarge = isTerminalInputTooLargeWithDeferredMeasurement(data) if (typeof tooLarge === 'boolean') { - return tooLarge ? false : writePtyProviderInputWithinLimit(provider, id, data) + return tooLarge ? false : writePtyProviderInputWithinLimit(provider, id, data, fence) } return tooLarge - .then((result) => (result ? false : writePtyProviderInputWithinLimit(provider, id, data))) + .then((result) => + result ? false : writePtyProviderInputWithinLimit(provider, id, data, fence) + ) .catch((error) => { reportUnavailablePtyWrite(id, error) return false @@ -7095,12 +7217,16 @@ export function registerPtyHandlers( id: string, chunks: Iterator, firstChunk: string, - secondChunk: string + secondChunk: string, + fence: PtyWriteFence ): Promise => { try { let chunk: IteratorResult = { done: false, value: firstChunk } let nextChunk: IteratorResult = { done: false, value: secondChunk } while (!chunk.done) { + if (!isCurrentPtyWrite(id, fence)) { + return false + } provider.write(id, chunk.value) if (!nextChunk.done) { await new Promise((resolve) => setTimeout(resolve, 0)) @@ -7145,11 +7271,17 @@ export function registerPtyHandlers( !mainWindow.isDestroyed() && !(typeof mainWebContents.isDestroyed === 'function' && mainWebContents.isDestroyed()) - const writePtyInput = (args: PtyWritePayload): boolean | Promise => { + const writePtyInput = ( + args: PtyWritePayload, + fence: PtyWriteFence + ): boolean | Promise => { // Why: mobile-presence-lock defense-in-depth — the renderer's onData guard can let one keystroke slip during the state-flip lag, so catch it server-side. See docs/mobile-presence-lock.md. if (runtime?.getDriver(args.id).kind === 'mobile') { return false } + if (!isCurrentPtyWrite(args.id, fence)) { + return false + } const provider = ptyOwnership.has(args.id) ? tryGetProviderForPty(args.id) : undefined if (!provider) { return false @@ -7161,16 +7293,22 @@ export function registerPtyHandlers( if (visibleRendererPtys.has(args.id)) { clearHiddenRendererResizeOutput(args.id) } - return writePtyProviderInput(provider, args.id, args.data) + return writePtyProviderInput(provider, args.id, args.data, fence) } catch { return false } } - const writePtyInputAccepted = (args: PtyWritePayload): boolean | Promise => { + const writePtyInputAccepted = ( + args: PtyWritePayload, + fence: PtyWriteFence + ): boolean | Promise => { if (runtime?.getDriver(args.id).kind === 'mobile') { return false } + if (!isCurrentPtyWrite(args.id, fence)) { + return false + } // Why: the ack infers Ctrl+C/Escape reached the local PTY; SSH providers are fire-and-forget relay notifications and can't truthfully acknowledge yet. if (ptyOwnership.get(args.id) !== null) { return false @@ -7186,7 +7324,7 @@ export function registerPtyHandlers( if (visibleRendererPtys.has(args.id)) { clearHiddenRendererResizeOutput(args.id) } - return writePtyProviderInput(provider, args.id, args.data) + return writePtyProviderInput(provider, args.id, args.data, fence) } catch { return false } @@ -7198,21 +7336,31 @@ export function registerPtyHandlers( if (!isPtyWriteEventFromMainWindow(event, mainWindow.webContents) || !isPtyWritePayload(args)) { return } - const claimTail = hostViewportClaimTails.get(args.id) - if (claimTail) { - void claimTail.then((claimed) => (claimed ? writePtyInput(args) : false)) + // Why here and not in the renderer: input queued before a reattach would + // otherwise land on whatever PTY now holds the pane. + const fence: PtyWriteFence = { incarnationId: ptyIncarnationById.get(args.id) } + if (!isCurrentPtyWrite(args.id, fence)) { return } - writePtyInput(args) + const claimTail = hostViewportClaimTails.get(args.id) + if (claimTail) { + void claimTail.then((claimed) => (claimed ? writePtyInput(args, fence) : false)) + return + } + writePtyInput(args, fence) }) ipcMain.handle('pty:writeAccepted', (event, args: unknown): boolean | Promise => { if (!isPtyWriteEventFromMainWindow(event, mainWindow.webContents) || !isPtyWritePayload(args)) { return false } + const fence: PtyWriteFence = { incarnationId: ptyIncarnationById.get(args.id) } + if (!isCurrentPtyWrite(args.id, fence)) { + return false + } const claimTail = hostViewportClaimTails.get(args.id) return claimTail - ? claimTail.then((claimed) => (claimed ? writePtyInputAccepted(args) : false)) - : writePtyInputAccepted(args) + ? claimTail.then((claimed) => (claimed ? writePtyInputAccepted(args, fence) : false)) + : writePtyInputAccepted(args, fence) }) ipcMain.removeAllListeners('pty:claimViewport') @@ -7245,6 +7393,10 @@ export function registerPtyHandlers( // Why: resize is fire-and-forget — ipcMain.on (not .handle) halves IPC traffic by skipping the empty acknowledgement reply. ipcMain.removeAllListeners('pty:resize') ipcMain.on('pty:resize', (_event, args: { id: string; cols: number; rows: number }) => { + // Why: a resize for a pane that has rebound would reshape the successor's shell. + if (isSupersededPtyId(args.id)) { + return + } // Why: after a desktop-fit override change the renderer's safeFit cascade re-measures ALL panes (background ones at full width), so suppress every pty:resize in this window to avoid corrupting PTY dimensions. if (runtime?.isResizeSuppressed()) { return @@ -7509,6 +7661,12 @@ export function registerPtyHandlers( ipcMain.removeAllListeners('pty:signal') ipcMain.on('pty:signal', (_event, args: { id: string; signal: string }) => { + // Why fenced but pty:kill is not: a signal means "interrupt MY pane", so a + // superseded id is a misdirected interrupt. A kill on a superseded id is the + // opposite — that PTY is now orphaned and reclaiming it is the point. + if (isSupersededPtyId(args.id)) { + return + } tryGetProviderForPty(args.id) ?.sendSignal(args.id, args.signal) .catch(() => {}) @@ -7661,6 +7819,9 @@ export function registerPtyHandlers( } const ownedConnectionId = ptyOwnership.get(args.id) const parsedSshId = ownedConnectionId === undefined ? parseAppSshPtyId(args.id) : null + // Why: before the cold-start daemon swap lands, the pre-swap local provider owns no + // daemon id and would answer an authoritative false for every restored session. + await getLocalPtyProviderStartupPromise(ownedConnectionId ?? parsedSshId?.connectionId) const provider = parsedSshId ? sshProviders.get(parsedSshId.connectionId) : tryGetProviderForPty(args.id) diff --git a/src/main/ipc/ssh-pty-output-intake.test.ts b/src/main/ipc/ssh-pty-output-intake.test.ts index be38872da04..43ceed26079 100644 --- a/src/main/ipc/ssh-pty-output-intake.test.ts +++ b/src/main/ipc/ssh-pty-output-intake.test.ts @@ -3,7 +3,7 @@ import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' import { createSshPtyOutputIntakeHarness as createHarness, sshPtyOutputEvent as event -} from './ssh-pty-output-intake-test-harness' +} from './__tests__/ssh-pty-output-intake-test-harness' describe('SshPtyOutputIntake', () => { it('plateaus at the model and pressure budgets, then resumes below low water', async () => { diff --git a/src/main/ipc/ssh-pty-output-model-migration.test.ts b/src/main/ipc/ssh-pty-output-model-migration.test.ts index ae56e2e45fc..59a063fba02 100644 --- a/src/main/ipc/ssh-pty-output-model-migration.test.ts +++ b/src/main/ipc/ssh-pty-output-model-migration.test.ts @@ -3,7 +3,7 @@ import type { SshPtyOutputDataEvent } from './ssh-pty-output-intake' import { createSshPtyOutputIntakeHarness as createHarness, sshPtyOutputEvent as event -} from './ssh-pty-output-intake-test-harness' +} from './__tests__/ssh-pty-output-intake-test-harness' // Production-shaped ids: bare relay id on the wire side, prefixed app id in intake. const APP_ID = 'ssh:conn@@pty-1' diff --git a/src/main/ipc/ssh.test.ts b/src/main/ipc/ssh.test.ts index 538cf4bf846..d4dcde25e4e 100644 --- a/src/main/ipc/ssh.test.ts +++ b/src/main/ipc/ssh.test.ts @@ -305,6 +305,7 @@ describe('SSH IPC handlers', () => { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), diff --git a/src/main/persistence-ssh-lease-termination.test.ts b/src/main/persistence-ssh-lease-termination.test.ts new file mode 100644 index 00000000000..c1c70c5c667 --- /dev/null +++ b/src/main/persistence-ssh-lease-termination.test.ts @@ -0,0 +1,152 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { getDefaultPersistedState } from '../shared/constants' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('./telemetry/client', () => ({ track: vi.fn() })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TARGET = 'ssh-target-1' + +type DurableFlushStore = { + flushDurableStateOrThrowAsync: (drainToStableGeneration?: boolean) => Promise +} + +function lease(ptyId: string, state: 'attached' | 'detached') { + return { + targetId: TARGET, + ptyId, + state, + createdAt: 1, + updatedAt: 1 + } +} + +async function createStore() { + mkdirSync(testState.dir, { recursive: true }) + writeFileSync( + join(testState.dir, 'orca-data.json'), + JSON.stringify({ + ...getDefaultPersistedState(testState.dir), + sshRemotePtyLeases: [ + lease('pty-1', 'detached'), + lease('pty-2', 'attached'), + lease('pty-3', 'detached') + ] + }), + 'utf-8' + ) + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() +} + +beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-ssh-lease-termination-')) +}) + +afterEach(() => { + rmSync(testState.dir, { force: true, recursive: true }) +}) + +describe('SSH lease termination persistence', () => { + it('updates only selected leases and awaits one asynchronous durability barrier', async () => { + const store = await createStore() + const asyncFlush = vi + .spyOn(store as unknown as DurableFlushStore, 'flushDurableStateOrThrowAsync') + .mockResolvedValue() + const syncFlush = vi.spyOn(store, 'flush') + const syncFlushOrThrow = vi.spyOn(store, 'flushOrThrow') + + await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1', 'pty-3']) + + expect(asyncFlush).toHaveBeenCalledOnce() + expect(asyncFlush).toHaveBeenCalledWith(false) + expect(syncFlush).not.toHaveBeenCalled() + expect(syncFlushOrThrow).not.toHaveBeenCalled() + expect(store.getSshRemotePtyLeases(TARGET)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }), + expect.objectContaining({ ptyId: 'pty-2', state: 'attached' }), + expect.objectContaining({ ptyId: 'pty-3', state: 'terminated' }) + ]) + ) + }) + + it('does not overwrite concurrent lease state when durability fails', async () => { + const store = await createStore() + const writeError = new Error('disk full') + const asyncFlush = vi + .spyOn(store as unknown as DurableFlushStore, 'flushDurableStateOrThrowAsync') + .mockRejectedValue(writeError) + const scheduleSave = vi.spyOn(store as unknown as { scheduleSave: () => void }, 'scheduleSave') + scheduleSave.mockClear() + + const retirement = store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1']) + store.markSshRemotePtyLeasesForShutdown(TARGET, 'detached') + + await expect(retirement).rejects.toBe(writeError) + expect(scheduleSave).toHaveBeenCalledTimes(2) + expect(store.getSshRemotePtyLeases(TARGET)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }), + expect.objectContaining({ ptyId: 'pty-2', state: 'detached' }) + ]) + ) + + asyncFlush.mockRestore() + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + const persisted = JSON.parse(readFileSync(join(testState.dir, 'orca-data.json'), 'utf-8')) as { + sshRemotePtyLeases: { ptyId: string; state: string }[] + } + expect(persisted.sshRemotePtyLeases).toContainEqual( + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }) + ) + }) + + it('does not write when no requested lease exists', async () => { + const store = await createStore() + const asyncFlush = vi.spyOn( + store as unknown as DurableFlushStore, + 'flushDurableStateOrThrowAsync' + ) + + await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['missing']) + + expect(asyncFlush).not.toHaveBeenCalled() + }) + + it('does not chase mutations made after the retirement generation is durable', async () => { + const store = await createStore() + const internal = store as unknown as { enqueueWrite: () => Promise } + const enqueueWrite = internal.enqueueWrite.bind(store) + let injectedMutation = false + const enqueueSpy = vi.spyOn(internal, 'enqueueWrite').mockImplementation(async () => { + await enqueueWrite() + if (!injectedMutation) { + injectedMutation = true + store.updateUI({ sidebarWidth: 777 }) + } + }) + + await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1']) + + expect(enqueueSpy).toHaveBeenCalledOnce() + const persisted = JSON.parse(readFileSync(join(testState.dir, 'orca-data.json'), 'utf-8')) as { + sshRemotePtyLeases: { ptyId: string; state: string }[] + } + expect(persisted.sshRemotePtyLeases).toContainEqual( + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }) + ) + enqueueSpy.mockRestore() + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) +}) diff --git a/src/main/persistence.test.ts b/src/main/persistence.test.ts index 17a73fa0e48..899aaa001f8 100644 --- a/src/main/persistence.test.ts +++ b/src/main/persistence.test.ts @@ -9992,6 +9992,9 @@ describe('Store', () => { expect(store.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey?.[paneKey]).toBe( 'inc-live' ) + // Unchanged by STA-3077 step P. The fold moves an SSH pane's incarnation only when it moves + // the binding it fences, and this fixture's tab exists solely in the SSH partition — so + // there is no second home to disagree with, nothing folds, and both halves stay together. const reloaded = await createStore() expect(reloaded.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey?.[paneKey]).toBe( 'inc-live' diff --git a/src/main/persistence.ts b/src/main/persistence.ts index f9fee511d39..14d50d4d297 100644 --- a/src/main/persistence.ts +++ b/src/main/persistence.ts @@ -37,6 +37,7 @@ import { nextAutomationOccurrenceAfter } from '../shared/automation-schedules' import { getAutomationLegacyRepoId } from '../shared/automation-run-identity' +import { isRelayAttestedPtyIncarnationId } from '../shared/pty-incarnation' import { normalizeAutomationPrecheck } from '../shared/automation-precheck' import { normalizeProxyUrl } from '../shared/network-proxy' import { normalizeKagiSessionLink } from '../shared/browser-url' @@ -96,6 +97,7 @@ import type { MigrationUnsupportedPtyEntry } from '../shared/agent-status-types' import { MOBILE_PAIRING_USERDATA_FILES } from './runtime/mobile-pairing-files' import { normalizePersistedMobileClientTabSelections } from './runtime/client-session-tab-selection-persistence' import { sanitizeWorkspaceSessionTerminalRetirements } from './runtime/mobile-session-terminal-persistence-retirement' +import { findTerminalTabIdForLeaf } from './runtime/workspace-session-terminal-membership-authority' import { removeRepoFromHostWorkspaceSessions, removeRepoFromWorkspaceSession @@ -1632,7 +1634,13 @@ function normalizeSshRemotePtyLease(value: unknown): SshRemotePtyLease | null { createdAt: typeof raw.createdAt === 'number' ? raw.createdAt : now, updatedAt: typeof raw.updatedAt === 'number' ? raw.updatedAt : now, ...(typeof raw.lastAttachedAt === 'number' ? { lastAttachedAt: raw.lastAttachedAt } : {}), - ...(typeof raw.lastDetachedAt === 'number' ? { lastDetachedAt: raw.lastDetachedAt } : {}) + ...(typeof raw.lastDetachedAt === 'number' ? { lastDetachedAt: raw.lastDetachedAt } : {}), + // Rebuilt field by field, so a lease is only as durable as this list — adding the field to the + // type without adding it here drops it on every load. Synthesized values are refused rather + // than carried: one written by an older build would later read as a different shell. + ...(isRelayAttestedPtyIncarnationId(raw.incarnationId) + ? { incarnationId: raw.incarnationId } + : {}) } } @@ -1733,6 +1741,20 @@ function layoutContainsLeafId(node: TerminalPaneLayoutNode | null, leafId: strin return layoutContainsLeafId(node.first, leafId) || layoutContainsLeafId(node.second, leafId) } +/** Total order so two leases for one pane resolve the same way on every host. */ +function isNewerSshRemotePtyLease( + candidate: SshRemotePtyLease, + incumbent: SshRemotePtyLease +): boolean { + if (candidate.updatedAt !== incumbent.updatedAt) { + return candidate.updatedAt > incumbent.updatedAt + } + if (candidate.createdAt !== incumbent.createdAt) { + return candidate.createdAt > incumbent.createdAt + } + return candidate.ptyId > incumbent.ptyId +} + function cloneLayoutNode(node: TerminalPaneLayoutNode): TerminalPaneLayoutNode { if (node.type === 'leaf') { return { type: 'leaf', leafId: node.leafId } @@ -2828,6 +2850,7 @@ export class Store { private pendingGithubCacheWrite: Promise | null = null private readonly staleGithubCacheTempCleanup: Promise private gitUsernameCache = new Map() + private readonly sshRemotePtyLeaseMutationVersions = new WeakMap() private readonly protectedSecrets = new ProtectedSecretPersistence() private loadNeedsSave = false private settingsChangeListeners = new Set< @@ -6841,10 +6864,14 @@ export class Store { incarnationId?: string startupCwd?: string expectedBinding?: { ptyId: string; incarnationId?: string } + /** Reattach passes false: an absent durable pane never authorizes creating UI. + * Defaults true so the spawn path keeps its force-quit-race branches. */ + mayCreate?: boolean expectedSourceBinding?: PtyBindingSourceExpectation }, hostId?: string | null ): boolean { + const mayCreate = args.mayCreate ?? true const resolvedHostId = this.resolveHostId(hostId) const session = this.getWorkspaceSession(resolvedHostId) const paneKey = `${args.tabId}:${args.leafId}` @@ -6957,6 +6984,10 @@ export class Store { [bindingWorktreeId]: session.activeTabIdByWorktree?.[bindingWorktreeId] ?? args.tabId } } + if (!mayCreate && terminalMembershipChanged) { + restoreSession() + return false + } if (!isTerminalLeafId(args.leafId)) { // Why: keep legacy renderer-local pane ids out of durable leaf-keyed layout state after the UUID migration. advanceTopologyFence() @@ -7007,6 +7038,10 @@ export class Store { } } } + if (!mayCreate && terminalMembershipChanged) { + restoreSession() + return false + } advanceTopologyFence() try { this.flushOrThrow() @@ -7302,6 +7337,12 @@ export class Store { return leases.filter((lease) => targetId === undefined || lease.targetId === targetId) } + private advanceSshRemotePtyLeaseMutationVersion(lease: SshRemotePtyLease): number { + const version = (this.sshRemotePtyLeaseMutationVersions.get(lease) ?? 0) + 1 + this.sshRemotePtyLeaseMutationVersions.set(lease, version) + return version + } + upsertSshRemotePtyLease( lease: Omit & Partial> @@ -7333,9 +7374,301 @@ export class Store { } else { this.state.sshRemotePtyLeases.push(next) } + this.supersedeSiblingLeasesForPane(next, now) this.flush() } + /** + * One pane owns at most one live remote PTY. Without this, lease identity is + * `(targetId, ptyId)` alone, so a pane that re-leases under a new relay id + * leaves its predecessor live forever — reattach then fans out over both and + * grafts a pane the user never opened (STA-3077: 2 -> 19 -> 20 across three + * reconnects). + * + * Superseded leases are marked `expired`, not terminated: the remote shell is + * deliberately left running, because losing a lease is not proof the shell died. + */ + private supersedeSiblingLeasesForPane(winner: SshRemotePtyLease, now: number): void { + if (!winner.worktreeId || !winner.tabId || !winner.leafId) { + return + } + if (winner.state === 'terminated' || winner.state === 'expired') { + return + } + // Why consult the binding here: at upsert time the caller's lease may not be + // the one the pane is bound to yet. Expiring the bound predecessor would + // detach a live pane, so leave both live and let reattach arbitrate with the + // binding in hand. + const boundPtyId = this.durablyBoundPtyIdForPane(winner.targetId, winner.tabId, winner.leafId) + if (boundPtyId && boundPtyId !== winner.ptyId) { + return + } + const superseded: SshRemotePtyLease[] = [] + for (const lease of this.state.sshRemotePtyLeases ?? []) { + if ( + lease.ptyId !== winner.ptyId && + lease.targetId === winner.targetId && + lease.worktreeId === winner.worktreeId && + // Leaf only: a lease freezes its tabId, and a pane broken out to a new tab would otherwise + // never compete with its own predecessor — which is the reported cardinality growth. + lease.leafId === winner.leafId + ) { + if (lease.state === 'expired') { + // A concurrent same-pane owner confirms this lease must outlive rollback as retired. + this.advanceSshRemotePtyLeaseMutationVersion(lease) + continue + } + if (lease.state === 'terminated') { + continue + } + lease.state = 'expired' + lease.updatedAt = now + this.advanceSshRemotePtyLeaseMutationVersion(lease) + superseded.push(lease) + } + } + // Why: matching on lease ptyId first means this scrubs only the predecessor's + // stale binding — the winner's own binding cannot match and is left intact. + this.clearSshRemotePtyBindingsForLeases(winner.targetId, superseded, 'local') + } + + /** The PTY a pane is durably bound to. The desktop plane's home is `local` — the renderer is its + * only publisher of pane membership. Hedging into `ssh:` let the headless plane's copy + * outvote the live binding and silently no-op supersession (STA-3077). + * + * Keyed on the leaf, falling back across tabs: a lease freezes its tabId at write time, and + * `detachTerminalPaneToTab` moves a live pane, so the named tab can be the one the pane left. + * Missing the binding there makes it lose to recency and retires the pane's own shell. */ + private durablyBoundPtyIdForPane( + targetId: string, + tabId: string, + leafId: string + ): string | undefined { + // Deliberately does NOT require the tab to still exist, unlike findTerminalTabIdForLeaf, which + // must — binding a live shell to a deleted tab registers a ghost pane. Here the consequence + // runs the other way: a binding under a departed tab only ever keeps a lease alive, and not + // retiring is the safe direction when nothing has proved the shell dead. + const findLeafBinding = (session: WorkspaceSessionState | undefined): string | undefined => { + const layouts = session?.terminalLayoutsByTabId + return ( + layouts?.[tabId]?.ptyIdsByLeafId?.[leafId] ?? + Object.values(layouts ?? {}).find((layout) => layout?.ptyIdsByLeafId?.[leafId]) + ?.ptyIdsByLeafId?.[leafId] + ) + } + // Local FIRST, and only then the headless plane. Local-first is what STA-3077 needed: the old + // code preferred `ssh:`, so a copy no live writer maintained outvoted the real binding + // and supersession no-opped. The fallback is not that hedge — it only speaks for a pane local + // says nothing about, so a headless-owned pane still gets a vote before its lease is retired. + const boundPtyId = + findLeafBinding(this.state.workspaceSession) ?? + findLeafBinding(this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)]) + return boundPtyId ? this.getRelayPtyIdForSshLeaseComparison(targetId, boundPtyId) : undefined + } + + /** + * The durable pane binding outranks recency. Picking the newest lease alone + * would retire the one the pane is actually bound to whenever a newer unbound + * lease exists, which detaches a live pane instead of healing it. + */ + private outranksForPane( + candidate: SshRemotePtyLease, + incumbent: SshRemotePtyLease, + targetId: string + ): boolean { + const boundPtyId = this.durablyBoundPtyIdForPane( + targetId, + candidate.tabId ?? '', + candidate.leafId ?? '' + ) + if (boundPtyId) { + if (incumbent.ptyId === boundPtyId) { + return false + } + if (candidate.ptyId === boundPtyId) { + return true + } + } + return isNewerSshRemotePtyLease(candidate, incumbent) + } + + /** + * Heals lease state that predates pane-keyed supersession. Existing installs + * carry the duplicates STA-3077 accumulated — one report reached 20 live + * leases for a handful of panes — and supersession alone only prevents new + * ones. Reattach calls this first so a stale predecessor cannot be revived. + * + * Returns the number of leases retired, for logging. + */ + async supersedeDuplicatePaneLeases(targetId: string): Promise { + const live = (this.state.sshRemotePtyLeases ?? []).filter( + (lease) => + lease.targetId === targetId && lease.state !== 'terminated' && lease.state !== 'expired' + ) + const winnerByPane = new Map() + for (const lease of live) { + if (!lease.worktreeId || !lease.tabId || !lease.leafId) { + continue + } + // Keyed on the leaf, not the frozen tabId: the same pane moved between tabs must land in one + // bucket, or its duplicates never arbitrate against each other. + const paneKey = [lease.worktreeId, lease.leafId].join('\0') + const incumbent = winnerByPane.get(paneKey) + if (!incumbent || this.outranksForPane(lease, incumbent, targetId)) { + winnerByPane.set(paneKey, lease) + } + } + if (winnerByPane.size === 0) { + return 0 + } + const winners = new Set(winnerByPane.values()) + const now = Date.now() + const superseded: SshRemotePtyLease[] = [] + const restore: (() => void)[] = [] + for (const lease of live) { + if (!lease.worktreeId || !lease.tabId || !lease.leafId || winners.has(lease)) { + continue + } + const { state, updatedAt } = lease + lease.state = 'expired' + lease.updatedAt = now + const rollbackVersion = this.advanceSshRemotePtyLeaseMutationVersion(lease) + restore.push(() => { + if ( + this.state.sshRemotePtyLeases?.includes(lease) && + this.sshRemotePtyLeaseMutationVersions.get(lease) === rollbackVersion && + lease.state === 'expired' && + lease.updatedAt === now + ) { + lease.state = state + lease.updatedAt = updatedAt + this.advanceSshRemotePtyLeaseMutationVersion(lease) + } + }) + superseded.push(lease) + } + if (superseded.length === 0) { + return 0 + } + const bindingsBefore = this.snapshotSshLeaseBindings(targetId, superseded) + this.clearSshRemotePtyBindingsForLeases(targetId, superseded, 'local') + this.scheduleSave() + try { + // Why the ASYNC twin: the sync flush fsyncs a multi-MB file from the Electron main thread, + // and this runs on every reconnect. On a stalled network profile mount that syscall is + // uninterruptible and nothing can bound it — see flushAsync. Why "OrThrow" and not plain + // flush(): flush() swallows write errors, which would leave these leases retired in memory + // but attached on disk for the rest of the session. + await this.flushDurableStateOrThrowAsync(false) + } catch (err) { + for (const undo of restore) { + undo() + } + this.restoreSshLeaseBindings(targetId, superseded, bindingsBefore) + this.scheduleSave() + console.error('[persistence] Failed to retire duplicate pane leases:', err) + return 0 + } + return superseded.length + } + + private snapshotSshLeaseBindings( + targetId: string, + leases: SshRemotePtyLease[] + ): { + tabs: { leafId: string; ptyId: string }[] + leaves: { leafId: string; ptyId: string }[] + } { + const tabSnapshots: { leafId: string; ptyId: string }[] = [] + const leaves: { leafId: string; ptyId: string }[] = [] + const session = this.state.workspaceSession + for (const [worktreeId, tabs] of Object.entries(session?.tabsByWorktree ?? {})) { + for (const tab of tabs) { + if (tab.ptyId) { + const lease = leases.find((candidate) => + this.sshRemotePtyLeaseMayReferenceBinding(candidate, { + ptyId: tab.ptyId!, + worktreeId, + targetId, + tabId: tab.id + }) + ) + if (lease?.leafId) { + tabSnapshots.push({ leafId: lease.leafId, ptyId: tab.ptyId }) + } + } + } + } + for (const [tabId, layout] of Object.entries(session?.terminalLayoutsByTabId ?? {})) { + const worktreeId = Object.entries(session?.tabsByWorktree ?? {}).find(([, tabs]) => + tabs.some((tab) => tab.id === tabId) + )?.[0] + for (const [leafId, ptyId] of Object.entries(layout.ptyIdsByLeafId ?? {})) { + if ( + leases.some( + (lease) => + lease.leafId === leafId && + this.sshRemotePtyLeaseMayReferenceBinding(lease, { + ptyId, + targetId, + ...(worktreeId ? { worktreeId } : {}), + tabId, + leafId + }) + ) + ) { + leaves.push({ leafId, ptyId }) + } + } + } + return { tabs: tabSnapshots, leaves } + } + + private restoreSshLeaseBindings( + targetId: string, + leases: SshRemotePtyLease[], + snapshots: { + tabs: { leafId: string; ptyId: string }[] + leaves: { leafId: string; ptyId: string }[] + } + ): void { + const currentLeases = new Set(this.state.sshRemotePtyLeases ?? []) + const session = this.state.workspaceSession + const hasLiveLease = (snapshot: { leafId: string; ptyId: string }): boolean => + leases.some( + (lease) => + currentLeases.has(lease) && + lease.state !== 'terminated' && + lease.state !== 'expired' && + lease.targetId === targetId && + lease.leafId === snapshot.leafId && + lease.ptyId === this.getRelayPtyIdForSshLeaseComparison(targetId, snapshot.ptyId) + ) + for (const snapshot of snapshots.tabs) { + const tabId = findTerminalTabIdForLeaf(session, snapshot.leafId) + const tab = Object.values(session?.tabsByWorktree ?? {}) + .flat() + .find((candidate) => candidate.id === tabId) + if (hasLiveLease(snapshot) && tab?.ptyId === null) { + tab.ptyId = snapshot.ptyId + } + } + for (const snapshot of snapshots.leaves) { + const tabId = findTerminalTabIdForLeaf(session, snapshot.leafId) + const layout = tabId ? session?.terminalLayoutsByTabId?.[tabId] : undefined + if ( + hasLiveLease(snapshot) && + layout && + layout.ptyIdsByLeafId?.[snapshot.leafId] === undefined + ) { + layout.ptyIdsByLeafId = { + ...layout.ptyIdsByLeafId, + [snapshot.leafId]: snapshot.ptyId + } + } + } + } + markSshRemotePtyLeases(targetId: string, state: SshRemotePtyLease['state']): void { if (this.updateSshRemotePtyLeaseStates(targetId, state)) { this.flush() @@ -7354,6 +7687,7 @@ export class Store { state: SshRemotePtyLease['state'] ): Promise { if (this.updateSshRemotePtyLeaseStates(targetId, state)) { + this.scheduleSave() await this.flushDurableStateOrThrowAsync() } } @@ -7366,10 +7700,30 @@ export class Store { ptyIds.map((ptyId) => this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)) ) if (this.updateSshRemotePtyLeaseStates(targetId, 'attached', relayPtyIds)) { + this.scheduleSave() await this.flushDurableStateOrThrowAsync() } } + async markSshRemotePtyLeasesTerminatedAsync( + targetId: string, + ptyIds: readonly string[] + ): Promise { + const relayPtyIds = new Set( + ptyIds.map((ptyId) => this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)) + ) + if (this.updateSshRemotePtyLeaseStates(targetId, 'terminated', relayPtyIds)) { + this.scheduleSave() + try { + await this.flushDurableStateOrThrowAsync(false) + } catch (error) { + // Keep the proven exit retryable after a transient persistence failure. + this.scheduleSave() + throw error + } + } + } + private updateSshRemotePtyLeaseStates( targetId: string, state: SshRemotePtyLease['state'], @@ -7388,8 +7742,13 @@ export class Store { continue } if (state === 'detached' && lease.state !== 'attached') { + if (lease.state === 'expired') { + // A later detach fences rollback from restoring attached ownership. + this.advanceSshRemotePtyLeaseMutationVersion(lease) + } continue } + this.advanceSshRemotePtyLeaseMutationVersion(lease) if (lease.state !== state) { lease.state = state lease.updatedAt = now @@ -7418,6 +7777,7 @@ export class Store { if (!lease) { return } + this.advanceSshRemotePtyLeaseMutationVersion(lease) const shouldClearBindings = state === 'terminated' || state === 'expired' if (lease.state === state) { if (shouldClearBindings && this.clearSshRemotePtyBindingsForLeases(targetId, [lease])) { @@ -7471,19 +7831,27 @@ export class Store { this.clearSshRemotePtyBindingsForLeases(targetId, leases ?? []) } + /** `arbitratedFrom` names the plane a supersession DECISION was made from. A decision reached by + * reading one plane may only mutate that plane: the other plane's binding never got a vote, and + * deleting it strands a shell its owner can still be using. An explicit expiry or termination is + * plane-agnostic — the pty is gone for everyone — so those callers pass nothing. */ private clearSshRemotePtyBindingsForLeases( targetId: string, - leases: SshRemotePtyLease[] + leases: SshRemotePtyLease[], + arbitratedFrom?: 'local' ): boolean { if (!leases?.length) { return false } let changed = false const sessions = new Set( - [ - this.state.workspaceSession, - this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)] - ].filter((session): session is WorkspaceSessionState => Boolean(session)) + (arbitratedFrom === 'local' + ? [this.state.workspaceSession] + : [ + this.state.workspaceSession, + this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)] + ] + ).filter((session): session is WorkspaceSessionState => Boolean(session)) ) for (const session of sessions) { for (const [worktreeId, tabs] of Object.entries(session.tabsByWorktree ?? {})) { @@ -7592,10 +7960,11 @@ export class Store { // Async twin of flushOrThrow: durable state only. Active-view and GitHub sidecars are // quit/startup work and must not be snapshotted on the live SSH establish/reconnect path. - private async flushDurableStateOrThrowAsync(): Promise { + private async flushDurableStateOrThrowAsync(drainToStableGeneration = true): Promise { if (this.writesFrozen || this.quitFlushStarted) { throw new Error('Cannot flush while persistence is finalized') } + const requiredDurableGeneration = this.writeGeneration for (;;) { if (this.writeTimer) { clearTimeout(this.writeTimer) @@ -7604,6 +7973,12 @@ export class Store { this.firstPendingSaveAt = null const generation = this.writeGeneration await this.enqueueWrite() + if ( + !drainToStableGeneration && + this.lastDurableWriteGeneration >= requiredDurableGeneration + ) { + break + } if (generation === this.writeGeneration) { break } diff --git a/src/main/providers/pty-provider-contract.ts b/src/main/providers/pty-provider-contract.ts index 09f3cda62fd..5e4a34db5a4 100644 --- a/src/main/providers/pty-provider-contract.ts +++ b/src/main/providers/pty-provider-contract.ts @@ -123,7 +123,8 @@ export type IPtyProvider = { /** Whether fresh structured creates can replay one spawn across a lost relay response. */ supportsAgentSessionCreateOperations?: (options?: PtyProbeOptions) => boolean | Promise attach(id: string): Promise | void> - hasPty?: (id: string) => boolean + /** Three-valued: null means the provider cannot see this id right now — absence is unproven, never dead. */ + hasPty?: (id: string) => boolean | null /** Exact provider readback: false only when the provider answered that the PTY is absent. */ probePtyLiveness?: (id: string) => Promise write(id: string, data: string): void diff --git a/src/main/providers/ssh-pty-errors.ts b/src/main/providers/ssh-pty-errors.ts index a8c551f228b..c389422f55d 100644 --- a/src/main/providers/ssh-pty-errors.ts +++ b/src/main/providers/ssh-pty-errors.ts @@ -1,12 +1,30 @@ -export const SSH_SESSION_EXPIRED_ERROR = 'SSH_SESSION_EXPIRED' -export const SSH_PTY_IDENTITY_MISMATCH_ERROR = 'SSH_PTY_IDENTITY_MISMATCH' +export { + SSH_SESSION_EXPIRED_ERROR, + SSH_PTY_IDENTITY_MISMATCH_ERROR, + SSH_PTY_EXITED_ERROR, + SSH_SOURCE_RESTORE_REQUIRED_ERROR +} from '../../shared/ssh-pty-failure-tokens' +import { + SSH_SOURCE_RESTORE_REQUIRED_ERROR, + isSshPtyExitedMessage, + isSshPtyIdentityMismatchMessage +} from '../../shared/ssh-pty-failure-tokens' + +export function isSshSourceRestoreRequiredError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error) + return message.includes(SSH_SOURCE_RESTORE_REQUIRED_ERROR) +} export function isSshPtyNotFoundError(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error) return /PTY ".+" not found/i.test(message) } +/** The relay watched this shell exit — first-hand proof, unlike a bare not-found. */ +export function isSshPtyExitedError(error: unknown): boolean { + return isSshPtyExitedMessage(error instanceof Error ? error.message : String(error)) +} + export function isSshPtyIdentityMismatchError(error: unknown): boolean { - const message = error instanceof Error ? error.message : String(error) - return message.includes(SSH_PTY_IDENTITY_MISMATCH_ERROR) || /identity mismatch/i.test(message) + return isSshPtyIdentityMismatchMessage(error instanceof Error ? error.message : String(error)) } diff --git a/src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts b/src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts new file mode 100644 index 00000000000..86d8cc36e58 --- /dev/null +++ b/src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts @@ -0,0 +1,215 @@ +/** + * An identity mismatch means the relay FOUND the PTY and its recorded pane + * differs — the shell is running. Publishing that as expiry makes the renderer + * clear the binding and cold-restore with agent resume, which is a second agent + * on one transcript while the first keeps running. + * + * Reachable today from a shipped gesture: detaching a pane into a new tab + * changes tabId, the relay still holds the tabId frozen at spawn, and the + * reattach mismatches. + * + * Pins the PRODUCER and the CONSUMERS together. The two guards this program + * shipped before both passed while sitting off the route production takes, so + * asserting the classifier alone is not enough — the destructive token has to + * be absent from what the reattach actually throws. + */ +import { describe, expect, it, vi } from 'vitest' +import { reattachSshPtySession } from './ssh-pty-session-reattach' +import { + SSH_SESSION_EXPIRED_ERROR, + SSH_PTY_IDENTITY_MISMATCH_ERROR, + formatPtyExitedError +} from '../../shared/ssh-pty-failure-tokens' + +/** The renderer converts a failure into `sessionExpired: true` on these tokens + * and then respawns on the flag alone, never consulting the classifier. + * Mirrors pty-transport.ts. */ +const SSH_PTY_CONNECTION_MISMATCH_MARKER = 'belongs to SSH connection' +function transportWouldReportSessionExpired(message: string): boolean { + return ( + message.includes(SSH_SESSION_EXPIRED_ERROR) || + message.includes(SSH_PTY_CONNECTION_MISMATCH_MARKER) + ) +} + +const CONNECTION_ID = 'ssh-target-1' +const RELAY_PTY_ID = 'pty-7' + +function muxThatFailsAttachWith(message: string): { mux: unknown } { + return { + mux: { + request: vi.fn().mockRejectedValue(new Error(message)), + notify: vi.fn() + } + } +} + +async function reattachError( + attachFailure: string, + expectedIncarnationId?: string +): Promise { + const { mux } = muxThatFailsAttachWith(attachFailure) + try { + await reattachSshPtySession({ + mux: mux as never, + connectionId: CONNECTION_ID, + sessionId: RELAY_PTY_ID, + options: { + cols: 80, + rows: 24, + paneKey: 'tab-new:leaf-1', + tabId: 'tab-new', + ...(expectedIncarnationId ? { expectedIncarnationId } : {}) + } as never + }) + } catch (error) { + return error as Error + } + throw new Error('reattach unexpectedly succeeded') +} + +describe('an identity mismatch is not a death', () => { + it('does not publish the destructive expiry token', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + }) + + // The renderer keys off this token; its own clauses live beside the + // classifier, in reattach-failure-classification.test.ts. + it('marks the failure with the shared mismatch token', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`) + + expect(error.message).toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR) + }) + + // The flag path bypasses the classifier entirely, so it needs its own clause. + it('does not trip the renderer transport into sessionExpired', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`) + + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) +}) + +describe('only an exit the relay watched is a death', () => { + // Clause-selectivity: a fix that silences real expiry would strand panes on a + // shell that truly went away, so the contrast has to hold. + it('publishes expiry when the relay reports the exit it observed', async () => { + const error = await reattachError( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-host-7'), + 'inc-host-7' + ) + + expect(error.message).toContain(SSH_SESSION_EXPIRED_ERROR) + expect(error.message).not.toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(true) + }) + + // INVERTED. This used to require a bare absence to publish expiry. That is the defect: the relay + // we asked cannot hand the id back, which proves an exit ONLY if it is the relay that minted it. + // A replaced relay answers exactly this for shells still running under its predecessor, so + // expiry there cleared ownership and resumed the agent a second time onto a live shell. The + // death that is real is still detected — by the clause above, from an exit the relay watched. + // Enforcement cannot live only on the host: the host is the party whose answer is in question, + // and versions differ. A proof we cannot tie to the shell we asked about is not proof. + it('does not publish expiry for an exit that names a different shell', async () => { + const error = await reattachError( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-some-other-shell'), + 'inc-host-7' + ) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) + + it('does not publish expiry for an exit that names a different PTY id', async () => { + const error = await reattachError( + formatPtyExitedError('pty-some-other-shell', 0, 'inc-host-7'), + 'inc-host-7' + ) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) + + it('does not publish expiry for an exit when the pane knows no incarnation', async () => { + const error = await reattachError(formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-host-7')) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + }) + + it('does not publish expiry when the relay merely has no such PTY', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found`) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) + + it('leaves an unrelated failure untouched rather than guessing', async () => { + const error = await reattachError('ECONNRESET while writing to the relay') + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(error.message).not.toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR) + }) +}) + +// New relays prioritize incarnation, so the pane fields do not reject a moved pane. Old relays +// ignore the additive incarnation field and still need their weaker pane fence. +describe('reattach preserves the old-relay pane fence', () => { + async function attachParams( + extraOptions: Record = {} + ): Promise> { + const request = vi.fn().mockRejectedValue(new Error('boom')) + try { + await reattachSshPtySession({ + mux: { request, notify: vi.fn() } as never, + connectionId: CONNECTION_ID, + sessionId: RELAY_PTY_ID, + options: { + cols: 80, + rows: 24, + paneKey: 'tab-new:leaf-1', + tabId: 'tab-new', + env: { ORCA_PANE_KEY: 'tab-old:leaf-1', ORCA_TAB_ID: 'tab-old' }, + ...extraOptions + } as never + }) + } catch { + // the attach failure is not what this clause is about + } + const call = request.mock.calls.at(0) + return (call?.[1] ?? call?.[0] ?? {}) as Record + } + + it('sends the current pane key for relays that do not understand incarnation', async () => { + expect(await attachParams()).toMatchObject({ expectedPaneKey: 'tab-new:leaf-1' }) + }) + + it('sends the current tab id for relays that do not understand incarnation', async () => { + expect(await attachParams()).toMatchObject({ expectedTabId: 'tab-new' }) + }) + + // The producer pin for the relay's incarnation guard. The guard is only worth having if the + // expectation actually leaves the client, and nothing else here would notice if it stopped: + // the relay stays permissive on an absent field, so a silent regression reads as "all green". + it('sends the expected incarnation when the host attested one', async () => { + expect(await attachParams({ expectedIncarnationId: 'inc-host-7' })).toMatchObject({ + expectedIncarnationId: 'inc-host-7', + expectedPaneKey: 'tab-new:leaf-1', + expectedTabId: 'tab-new' + }) + }) + + // A stand-in minted locally when the host reported none. It is first-write-wins and is dropped + // when provider state resets, so the same live shell can present a different one after a + // reconnect — sending it would make the relay refuse the pane its own shell. + it('sends no expected incarnation when the value was synthesized locally', async () => { + expect(await attachParams({ expectedIncarnationId: 'legacy:23:0:pty-7' })).not.toHaveProperty( + 'expectedIncarnationId' + ) + }) + + it('sends no expected incarnation when there is none to send', async () => { + expect(await attachParams()).not.toHaveProperty('expectedIncarnationId') + }) +}) diff --git a/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts b/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts index e088b21f025..b150c7fa0c7 100644 --- a/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts +++ b/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts @@ -316,7 +316,8 @@ describe('SSH fresh agent-session create operations', () => { ) expect(onData).not.toHaveBeenCalled() - expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(false) + // Not `false`: the provider has never listed this host, so absence is unknown, not proven. + expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).not.toBe(true) const shutdownRequest = await waitForRequest(transport, 'pty.shutdown') transport.deliver(responseFrame(shutdownRequest.id as number, null, 4)) const cancelRequest = await waitForRequest(transport, 'pty.cancelDelivery') diff --git a/src/main/providers/ssh-pty-provider-exit-race.test.ts b/src/main/providers/ssh-pty-provider-exit-race.test.ts index 22a19985d21..38c665d93c4 100644 --- a/src/main/providers/ssh-pty-provider-exit-race.test.ts +++ b/src/main/providers/ssh-pty-provider-exit-race.test.ts @@ -170,3 +170,30 @@ it('returns a provisional source activation lease to reconnect authority', async deliveryToken: 'token-reconnect' }) }) + +it('fences a mismatched attach response before installing its source activation', async () => { + const response = { + incarnationId: 'incarnation-other', + sourceActivation: sourceActivation('incarnation-other') + } + const mux = { + request: vi.fn(async (method: string, _params, options) => { + if (method !== 'pty.attach') { + return undefined + } + options?.beforeResolve?.(response) + return response + }), + notify: vi.fn(), + onNotification: vi.fn(), + dispose: vi.fn(), + isDisposed: vi.fn().mockReturnValue(false) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.attachForReconnect('ssh:conn-1@@pty-1', undefined, 'incarnation-expected') + ).rejects.toThrow('SSH_PTY_IDENTITY_MISMATCH') + + expect(mux.request).toHaveBeenCalledTimes(1) +}) diff --git a/src/main/providers/ssh-pty-provider-output-state.ts b/src/main/providers/ssh-pty-provider-output-state.ts index 4d98097b9c6..a80ffc4ec4f 100644 --- a/src/main/providers/ssh-pty-provider-output-state.ts +++ b/src/main/providers/ssh-pty-provider-output-state.ts @@ -11,6 +11,7 @@ import { type SshPtyReceivingActivationLease } from './ssh-pty-notification-routing' import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' +import { SYNTHESIZED_PTY_INCARNATION_PREFIX } from '../../shared/pty-incarnation' export class SshPtyProviderOutputState { private readonly dataListeners = new Set() @@ -132,7 +133,7 @@ export class SshPtyProviderOutputState { this.rememberPtyIncarnation(relayPtyId, incarnationId) let resolved = this.incarnationByRelayPtyId.get(relayPtyId) if (!resolved) { - resolved = `legacy:${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}` + resolved = `${SYNTHESIZED_PTY_INCARNATION_PREFIX}${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}` this.incarnationByRelayPtyId.set(relayPtyId, resolved) } return resolved diff --git a/src/main/providers/ssh-pty-provider-process-events.test.ts b/src/main/providers/ssh-pty-provider-process-events.test.ts index e152ef47fae..449c3e4c356 100644 --- a/src/main/providers/ssh-pty-provider-process-events.test.ts +++ b/src/main/providers/ssh-pty-provider-process-events.test.ts @@ -29,6 +29,17 @@ describe('SshPtyProvider process listings and events', () => { provider = new SshPtyProvider('conn-1', mux as never) }) + // Why: a reconnect builds a fresh provider with an empty set, so a miss before the + // first completed listing is ignorance about the host, not a dead PTY. + it('answers unknown liveness until a completed listing makes absence provable', async () => { + expect(provider.hasPty(scopedPty1)).toBe(null) + + mux.request.mockResolvedValueOnce([]) + await provider.listProcesses() + + expect(provider.hasPty(scopedPty1)).toBe(false) + }) + it('scopes process listings and bounds the relay RPC by the teardown deadline', async () => { const processes = [{ id: 'pty-1', cwd: '/home', title: 'zsh', worktreeId: 'repo::/home' }] mux.request.mockResolvedValue(processes) diff --git a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts index 74989dc34e5..af7eeb98139 100644 --- a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts +++ b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts @@ -1,8 +1,28 @@ import { describe, expect, it, vi } from 'vitest' -import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' +import { + SSH_PTY_IDENTITY_MISMATCH_ERROR, + SSH_SESSION_EXPIRED_ERROR, + SSH_SOURCE_RESTORE_REQUIRED_ERROR +} from './ssh-pty-errors' import { SshPtyProvider } from './ssh-pty-provider' describe('SSH PTY provider session reattach incarnation', () => { + it.each([ + ['an old-relay response without an incarnation', {}], + ['a response for a different incarnation', { incarnationId: 'incarnation-other' }] + ])('rejects %s', async (_label, response) => { + const mux = { + request: vi.fn().mockResolvedValue(response), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.attachForReconnect('ssh:conn-1@@pty-old', undefined, 'incarnation-expected') + ).rejects.toThrow(SSH_PTY_IDENTITY_MISMATCH_ERROR) + }) + it('remembers the authoritative incarnation before a legacy exit arrives', async () => { let notify: ((method: string, params: Record) => void) | undefined const mux = { @@ -44,8 +64,10 @@ describe('SSH PTY provider session reattach incarnation', () => { } const provider = new SshPtyProvider('conn-1', mux as never) - await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( - `${SSH_SESSION_EXPIRED_ERROR}: pty-old` - ) + // Fails closed, but NOT as expiry: the shell is still running, and callers + // respawn on expiry — which duplicate-resumed the live agent session. + const spawn = provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) + await expect(spawn).rejects.toThrow(`${SSH_SOURCE_RESTORE_REQUIRED_ERROR}: pty-old`) + await expect(spawn).rejects.not.toThrow(SSH_SESSION_EXPIRED_ERROR) }) }) diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 40301906f52..e0b1d6b8213 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' +import { formatPtyExitedError } from '../../shared/ssh-pty-failure-tokens' import { SshPtyProvider } from './ssh-pty-provider' import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env' import { PTY_STARTUP_INGRESS_VERSION } from '../../shared/pty-startup-ingress' @@ -584,7 +585,8 @@ describe('SshPtyProvider', () => { id: 'pty-old', cols: 80, rows: 24, - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true }, sourceActivationRequestOptions ) @@ -630,7 +632,8 @@ describe('SshPtyProvider', () => { id: 'pty-old', cols: 80, rows: 24, - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true }) expect(result).toEqual({ id: 'ssh:conn-1@@pty-old', @@ -639,7 +642,8 @@ describe('SshPtyProvider', () => { }) }) - it('reattaches with explicit pane identity when hook env was stripped', async () => { + // New relays prioritize incarnation; old relays ignore that field and retain this weaker fence. + it('preserves pane identity for relays that predate incarnation fencing', async () => { mux.request.mockResolvedValue({ replay: 'buffered-output' }) await provider.spawn({ @@ -655,18 +659,30 @@ describe('SshPtyProvider', () => { cols: 80, rows: 24, suppressReplayNotification: true, + exitProofSupported: true, expectedPaneKey: 'tab-a:leaf-a', expectedTabId: 'tab-a' }) }) - it('does not fresh-spawn over an expired reattach session', async () => { - mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found')) - - await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( - 'SSH_SESSION_EXPIRED: pty-old' + it('does not fresh-spawn over a reattach whose shell the relay saw exit', async () => { + // INVERTED to drive the exit the relay OBSERVED. A bare not-found no longer becomes expiry — + // it proves nothing when the answering relay may be a replacement — so the observed exit is + // now what carries this through to expiry. The property under test is unchanged: a failed + // reattach throws instead of quietly spawning a second shell over the pane. + mux.request.mockRejectedValueOnce( + new Error(formatPtyExitedError('pty-old', 0, 'inc-host-old')) ) + await expect( + provider.spawn({ + cols: 80, + rows: 24, + sessionId: 'pty-old', + expectedIncarnationId: 'inc-host-old' + }) + ).rejects.toThrow('SSH_SESSION_EXPIRED: pty-old') + expect(mux.request).toHaveBeenNthCalledWith( 1, 'pty.attach', @@ -674,13 +690,27 @@ describe('SshPtyProvider', () => { id: 'pty-old', cols: 80, rows: 24, - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: 'inc-host-old' }, sourceActivationRequestOptions ) expect(mux.request).toHaveBeenCalledTimes(1) }) + // The sibling case, and the one the change is for: an id the relay simply does not know may be + // a shell still running under a relay this one replaced. It must still refuse to fresh-spawn, + // and must NOT be dressed up as expiry, which is what authorized the replacement. + it('does not fresh-spawn, nor claim expiry, when the relay merely has no such PTY', async () => { + mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found')) + + await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( + 'PTY "pty-old" not found' + ) + expect(mux.request).toHaveBeenCalledTimes(1) + }) + it('preserves transient reattach failures for retry handling', async () => { mux.request.mockRejectedValueOnce(new Error('SSH connection lost, reconnecting...')) @@ -714,7 +744,8 @@ describe('SshPtyProvider', () => { 'pty.attach', { id: 'pty-1', - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true }, expect.objectContaining({ timeoutMs: 10_000, @@ -739,28 +770,6 @@ describe('SshPtyProvider', () => { ) }) - it('attachForReconnect forwards expected identity when provided', async () => { - await provider.attachForReconnect(scopedPty1, { - paneKey: 'tab-a:leaf-a', - tabId: 'tab-a' - }) - - expectRequest( - mux.request, - 'pty.attach', - { - id: 'pty-1', - suppressReplayNotification: true, - expectedPaneKey: 'tab-a:leaf-a', - expectedTabId: 'tab-a' - }, - expect.objectContaining({ - timeoutMs: 10_000, - beforeResolve: expect.any(Function) - }) - ) - }) - it('write sends pty.data notification', () => { provider.write(scopedPty1, 'hello') expect(mux.notify).toHaveBeenCalledWith('pty.data', { id: 'pty-1', data: 'hello' }) diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 2bc52fe491d..2eaf202f0b2 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -13,6 +13,7 @@ import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state' import { spawnFreshSshPty } from './ssh-agent-session-create-operation' import { mapSshPtyProcessList } from './ssh-agent-session-process-list' import { + buildSshPtyReconnectAttachParams, requestSshPtyAttach, reattachSshPtySessionWithExitFence, type PtySourceRecoveryRequest, @@ -22,7 +23,7 @@ import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request' import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities' import type { PtyProcessInspection } from './pty-process-inspection' -import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' +import { SSH_SOURCE_RESTORE_REQUIRED_ERROR } from './ssh-pty-errors' // Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch. function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined { @@ -34,6 +35,7 @@ export class SshPtyProvider implements IPtyProvider { private mux: SshChannelMultiplexer private connectionId: string private livePtyIds = new Set() + private listedOnce = false readonly getAppliedSize: NonNullable private readonly agentSessionCapabilities: SshAgentSessionCapabilities private spawnExitRaces = new SshPtySpawnExitRaceTracker() @@ -63,6 +65,7 @@ export class SshPtyProvider implements IPtyProvider { dispose(): void { this.outputState.dispose() this.livePtyIds.clear() + this.listedOnce = false } getConnectionId = (): string => this.connectionId @@ -101,8 +104,11 @@ export class SshPtyProvider implements IPtyProvider { this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId) }) if (result.sourceRecovery?.status === 'restoreRequired') { + // Why not SSH_SESSION_EXPIRED: the shell is still running, only its + // output source needs re-establishing. Reporting expiry made the pane + // respawn and resume the same agent session twice into one transcript. throw new Error( - `${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}` + `${SSH_SOURCE_RESTORE_REQUIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}` ) } this.livePtyIds.add(result.id) @@ -177,20 +183,16 @@ export class SshPtyProvider implements IPtyProvider { async attachForReconnect( id: string, - expected?: { paneKey?: string; tabId?: string }, - sourceRecovery?: PtySourceRecoveryRequest + sourceRecovery?: PtySourceRecoveryRequest, + expectedIncarnationId?: string, + legacyExpectedIdentity?: { paneKey?: string; tabId?: string } ): Promise { - // Why: reconnect owns replay delivery so stale/duplicate attach results can - // be filtered before they reach the renderer. The expected identity lets the - // relay reject a cross-generation id collision instead of reattaching this - // lease to a different pane's freshly spawned PTY. - const params = { + const params = buildSshPtyReconnectAttachParams({ id: this.toRelayPtyId(id), - suppressReplayNotification: true, ...(sourceRecovery ? { sourceRecovery } : {}), - ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), - ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) - } + ...(expectedIncarnationId ? { expectedIncarnationId } : {}), + ...(legacyExpectedIdentity ? { legacyExpectedIdentity } : {}) + }) const relayPtyId = this.toRelayPtyId(id) return await requestSshPtyAttach({ mux: this.mux, @@ -296,11 +298,14 @@ export class SshPtyProvider implements IPtyProvider { const relayPtyId = this.toRelayPtyId(process.id) this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId) } + this.listedOnce = true return processes } - hasPty(id: string): boolean { - return this.livePtyIds.has(id) + hasPty(id: string): boolean | null { + // Why null before a completed listing: a reconnect builds a new provider with an + // empty set, so a miss there is ignorance about the host, not a dead PTY. + return this.livePtyIds.has(id) ? true : this.listedOnce ? false : null } async getDefaultShell(): Promise { diff --git a/src/main/providers/ssh-pty-session-reattach.ts b/src/main/providers/ssh-pty-session-reattach.ts index 34974940caf..ff490464853 100644 --- a/src/main/providers/ssh-pty-session-reattach.ts +++ b/src/main/providers/ssh-pty-session-reattach.ts @@ -1,11 +1,16 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' -import { isPtyIncarnationId, type PtyIncarnationId } from '../../shared/pty-incarnation' +import { + isPtyIncarnationId, + isRelayAttestedPtyIncarnationId, + type PtyIncarnationId +} from '../../shared/pty-incarnation' import { SSH_PTY_IDENTITY_MISMATCH_ERROR, SSH_SESSION_EXPIRED_ERROR, - isSshPtyIdentityMismatchError, - isSshPtyNotFoundError + isSshPtyExitedError, + isSshPtyIdentityMismatchError } from './ssh-pty-errors' +import { parseMatchingPtyExitedError } from '../../shared/ssh-pty-failure-tokens' import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' import type { PtySpawnOptions, PtySpawnResult } from './types' import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' @@ -27,6 +32,29 @@ export type SshPtyAttachResult = { sourceActivationLease?: SshPtyReceivingActivationLease } +export function buildSshPtyReconnectAttachParams(args: { + id: string + sourceRecovery?: PtySourceRecoveryRequest + expectedIncarnationId?: string + legacyExpectedIdentity?: { paneKey?: string; tabId?: string } +}): Record { + return { + id: args.id, + suppressReplayNotification: true, + exitProofSupported: true, + ...(isRelayAttestedPtyIncarnationId(args.expectedIncarnationId) + ? { expectedIncarnationId: args.expectedIncarnationId } + : {}), + ...(args.legacyExpectedIdentity?.paneKey + ? { expectedPaneKey: args.legacyExpectedIdentity.paneKey } + : {}), + ...(args.legacyExpectedIdentity?.tabId + ? { expectedTabId: args.legacyExpectedIdentity.tabId } + : {}), + ...(args.sourceRecovery ? { sourceRecovery: args.sourceRecovery } : {}) + } +} + type SshPtyReattachResult = PtySpawnResult & { sourceRecovery?: PtySourceRecoveryResult sourceActivationLease?: SshPtyReceivingActivationLease @@ -85,6 +113,18 @@ export async function requestSshPtyAttach(args: { rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void }): Promise { let activationLease: SshPtyReceivingActivationLease | undefined + const parseResult = (value: unknown): SshPtyAttachResult => { + const result = parseSshPtyAttachResult(value) + const expectedIncarnationId = args.params.expectedIncarnationId + // Old relays ignore the request fence, so their response must prove the same shell. + if ( + isRelayAttestedPtyIncarnationId(expectedIncarnationId) && + result.incarnationId !== expectedIncarnationId + ) { + throw new Error(`${SSH_PTY_IDENTITY_MISMATCH_ERROR}: ${args.relayPtyId}`) + } + return result + } const installFromResult = (result: SshPtyAttachResult): void => { if (!activationLease && result.sourceActivation && args.installSourceActivation) { activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation) @@ -93,9 +133,9 @@ export async function requestSshPtyAttach(args: { try { const rawResult = await args.mux.request('pty.attach', args.params, { ...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }), - beforeResolve: (value) => installFromResult(parseSshPtyAttachResult(value)) + beforeResolve: (value) => installFromResult(parseResult(value)) }) - const result = parseSshPtyAttachResult(rawResult) + const result = parseResult(rawResult) installFromResult(result) args.rememberPtyIncarnation?.(args.relayPtyId, result.incarnationId) if (args.commitSourceActivation) { @@ -185,7 +225,6 @@ export async function reattachSshPtySession(args: { const relaySessionId = toRelaySshPtyId(args.connectionId, args.sessionId) console.warn(`[ssh-pty] spawn() called with sessionId=${args.sessionId}, attempting pty.attach`) try { - // Why: expected pane identity prevents a reused relay id from attaching the wrong shell. const expectedPaneKey = args.options.paneKey ?? args.options.env?.ORCA_PANE_KEY const expectedTabId = args.options.tabId ?? args.options.env?.ORCA_TAB_ID const attachResult = await requestSshPtyAttach({ @@ -196,6 +235,17 @@ export async function reattachSshPtySession(args: { cols: args.options.cols, rows: args.options.rows, suppressReplayNotification: true, + // Declares that this client can act on a proven exit. Without it a host must keep answering + // in the older wording, because an older client reads anything else as an unknown failure + // and leaves the pane with no shell and no way back. + exitProofSupported: true, + // The shell's own identity, so it survives a pane moving between tabs — unlike the pane + // identity this replaced. Sent only when the host attested it: a locally synthesized + // stand-in is not stable across reconnects and would refuse the pane its own shell. An + // older relay ignores the field, so the legacy pane fence below remains its fallback. + ...(isRelayAttestedPtyIncarnationId(args.options.expectedIncarnationId) + ? { expectedIncarnationId: args.options.expectedIncarnationId } + : {}), ...(expectedPaneKey ? { expectedPaneKey } : {}), ...(expectedTabId ? { expectedTabId } : {}) }, @@ -219,12 +269,34 @@ export async function reattachSshPtySession(args: { } catch (error) { // Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding. console.warn(`[ssh-pty] pty.attach FAILED for ${args.sessionId}:`, error) - if (isSshPtyNotFoundError(error)) { - const mismatchMarker = isSshPtyIdentityMismatchError(error) - ? ` ${SSH_PTY_IDENTITY_MISMATCH_ERROR}` - : '' - throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}${mismatchMarker}`) + // Why: the relay reports a mismatch by saying "not found", but it found the + // pty — comparing identity is how it knows. Publishing expiry there makes + // the renderer respawn and resume the agent a second time onto a live shell. + if (isSshPtyIdentityMismatchError(error)) { + throw new Error(`${SSH_PTY_IDENTITY_MISMATCH_ERROR}: ${relaySessionId}`) } + // The relay WATCHED this shell exit, which is the only answer that proves it is gone, so this + // is the one route that may authorize a replacement. The proof still has to be about OUR shell: + // the host applies that rule too, but the host is the party whose answer is in question and + // versions differ, so a proof we cannot tie to the incarnation we asked about is not proof and + // falls through to the disconnected pane instead of replacing a shell that may be running. + if (isSshPtyExitedError(error)) { + const proof = parseMatchingPtyExitedError( + error instanceof Error ? error.message : String(error), + relaySessionId, + args.options.expectedIncarnationId + ) + if (proof) { + throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}`) + } + throw error + } + // A bare not-found deliberately does NOT become expiry any more. It means the relay we asked + // cannot hand the id back, which is proof of an exit only if that relay is the one that minted + // it — and a replaced relay answers exactly this for shells still running under its + // predecessor. Treating it as death cleared ownership and resumed the agent a second time onto + // a live shell. Unproven now falls through to the caller, which shows the pane as + // disconnected and lets the user decide. throw error } } diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index aa68dad27ae..9d141068caa 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -2848,7 +2848,9 @@ describe('OrcaRuntimeService', () => { ) }) - it('recovers a disconnected pane through one HUB-owned replacement', async () => { + // Inverted for STA-3077 S8: the replacement grant this pinned was unreachable in production + // (relay-native lease ptyId vs app-form runtime ptyId) and has been deleted. + it('refuses to replace a disconnected SSH pane, even inside the lease grace window', async () => { const tabId = 'tab-recover' const runtime = createRuntimeWithSshLease('pty-expired', tabId) const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) @@ -2870,18 +2872,8 @@ describe('OrcaRuntimeService', () => { await expect( runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) - ).resolves.toMatchObject({ - handle: 'term-replacement', - tabId, - leafId: HEADLESS_LEAF_ID, - worktreeId: TEST_WORKTREE_ID - }) - expect(createTerminal).toHaveBeenCalledWith(`id:${TEST_WORKTREE_ID}`, { - tabId, - leafId: HEADLESS_LEAF_ID, - focus: false, - persistHostSessionBinding: true - }) + ).rejects.toThrow('terminal_not_recoverable') + expect(createTerminal).not.toHaveBeenCalled() }) it('rejects missing host panes without authoritative expired binding evidence', async () => { @@ -2947,7 +2939,9 @@ describe('OrcaRuntimeService', () => { expect(createTerminal).not.toHaveBeenCalled() }) - it('deduplicates concurrent pane recovery across stale viewer handles', async () => { + // Inverted for STA-3077 S8: this was the only consumer of the recovery-dedup map, which existed + // solely to de-duplicate the (unreachable, now deleted) replacement grant. Scenario kept. + it('concurrent recovery requests for one disconnected pane both refuse and create nothing', async () => { const tabId = 'tab-concurrent' const runtime = createRuntimeWithSshLease('pty-expired', tabId) const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) @@ -2957,30 +2951,23 @@ describe('OrcaRuntimeService', () => { }) const expiredHandle = runtime.resolveTerminalPane(paneKey, TEST_WORKTREE_ID).handle runtime.onPtyExit('pty-expired', 0) - let finishCreate!: (result: RuntimeTerminalCreate) => void - const pendingCreate = new Promise((resolve) => { - finishCreate = resolve - }) - const createTerminal = vi.spyOn(runtime, 'createTerminal').mockReturnValue(pendingCreate) + // Never settles: a resurrected grant would hang the assertion rather than quietly pass. + const createTerminal = vi + .spyOn(runtime, 'createTerminal') + .mockReturnValue(new Promise(() => {})) const first = runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) const second = runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, 'term-other-viewer') - finishCreate({ - handle: 'term-replacement', - tabId, - paneKey, - ptyId: 'pty-replacement', - worktreeId: TEST_WORKTREE_ID, - title: null, - surface: 'background' - }) - await expect(first).resolves.toEqual(expect.objectContaining({ handle: 'term-replacement' })) + await expect(first).rejects.toThrow('terminal_not_recoverable') + // The second viewer's handle never mapped to this pane, so it still fails the identity guard. await expect(second).rejects.toThrow('terminal_not_found') - expect(createTerminal).toHaveBeenCalledOnce() + expect(createTerminal).not.toHaveBeenCalled() }) - it('clears a failed pane recovery so a later reconnect can retry', async () => { + // Inverted for STA-3077 S8: it pinned retry-after-a-failed-grant, and the grant (unreachable at + // the pty-id shapes production mints) is deleted, so there is nothing left to retry into. + it('a repeated recovery request never accumulates a grant', async () => { const tabId = 'tab-retry' const runtime = createRuntimeWithSshLease('pty-expired', tabId) const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) @@ -2990,6 +2977,7 @@ describe('OrcaRuntimeService', () => { }) const expiredHandle = runtime.resolveTerminalPane(paneKey, TEST_WORKTREE_ID).handle runtime.onPtyExit('pty-expired', 0) + // A fully working createTerminal stays armed across both calls to prove neither one reaches it. const createTerminal = vi .spyOn(runtime, 'createTerminal') .mockRejectedValueOnce(new Error('relay_reconnecting')) @@ -3005,13 +2993,16 @@ describe('OrcaRuntimeService', () => { await expect( runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) - ).rejects.toThrow('relay_reconnecting') + ).rejects.toThrow('terminal_not_recoverable') await expect( runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) - ).resolves.toMatchObject({ handle: 'term-retry' }) - expect(createTerminal).toHaveBeenCalledTimes(2) + ).rejects.toThrow('terminal_not_recoverable') + expect(createTerminal).toHaveBeenCalledTimes(0) }) + // STA-3077 S8: no longer discriminating — since the grant was deleted, an 'expired' lease refuses + // identically (see the grace-window case above), so 'terminated' no longer selects the outcome. + // Retained only as a regression guard: if a replacement path ever returns, this must stay a refusal. it('does not recover a pane whose authoritative SSH lease was terminated', async () => { const tabId = 'tab-terminated' const runtime = createRuntimeWithSshLease('pty-terminated', tabId, 'terminated') diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 55d2195c31e..856b5960b32 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2771,7 +2771,6 @@ export class OrcaRuntimeService { } >() private terminalSleepGeneration = 0 - private terminalPaneRecoveryByIdentity = new Map>() // Why: idempotency map for worktree.create — a create interrupted by a mobile // connection migration is retried with the same clientMutationId and returns // the in-flight (or just-finished) operation instead of a duplicate worktree. @@ -6211,8 +6210,7 @@ export class OrcaRuntimeService { private getRecentExpiredSshLease( worktreeId: string, tabId: string, - leafId: string | undefined, - ptyId?: string + leafId: string | undefined ): ReturnType>[number] | null { const now = Date.now() return ( @@ -6223,7 +6221,6 @@ export class OrcaRuntimeService { lease.state === 'expired' && lease.worktreeId === worktreeId && lease.tabId === tabId && - (ptyId === undefined || lease.ptyId === ptyId) && (leafId === undefined || lease.leafId === undefined || lease.leafId === leafId) && lease.updatedAt <= now && now - lease.updatedAt <= SSH_PANE_RECOVERY_GRACE_MS @@ -16525,46 +16522,17 @@ export class OrcaRuntimeService { ) { throw new Error('terminal_not_found') } - const recoveryKey = `${expectedWorktreeId}\0${paneKey}` - const pending = this.terminalPaneRecoveryByIdentity.get(recoveryKey) - if (pending) { - return pending - } if (pty?.connected) { const current = this.resolveTerminalPane(paneKey, expectedWorktreeId) if (expectedHandle === undefined || current.handle !== expectedHandle) { return current } - throw new Error('terminal_not_recoverable') } - if ( - !this.getRecentExpiredSshLease(expectedWorktreeId, parsed.tabId, parsed.leafId, pty.ptyId) - ) { - // Why: an explicit close leaves a terminated lease; only relay expiry authorizes shell recreation. - throw new Error('terminal_not_recoverable') - } - // Why: disconnected PTYs can reissue handles during graph cleanup; only a connected replacement satisfies the pane CAS. - const recovery = this.createTerminal(`id:${expectedWorktreeId}`, { - tabId: parsed.tabId, - leafId: parsed.leafId, - focus: false, - // Why: the HUB renderer may publish its exited layout while recovery is in flight; persist the replacement before that stale graph can orphan it. - persistHostSessionBinding: true - }).then((terminal) => ({ - handle: terminal.handle, - tabId: parsed.tabId, - leafId: parsed.leafId, - ptyId: terminal.ptyId ?? null, - worktreeId: expectedWorktreeId - })) - this.terminalPaneRecoveryByIdentity.set(recoveryKey, recovery) - const clearRecovery = (): void => { - if (this.terminalPaneRecoveryByIdentity.get(recoveryKey) === recovery) { - this.terminalPaneRecoveryByIdentity.delete(recoveryKey) - } - } - void recovery.then(clearRecovery, clearRecovery) - return recovery + // A disconnected pane is never silently replaced. The grant this used to hold could not fire: + // it compared a relay-native lease id against an app-form runtime id (STA-3077 S6/S8, oracle in + // ssh-pane-recovery-grant-reachability.test.ts). The pane surfaces as disconnected instead, and + // spawning a replacement is the user's explicit choice. + throw new Error('terminal_not_recoverable') } async showTerminal(handle: string): Promise { @@ -30128,13 +30096,20 @@ export class OrcaRuntimeService { // The sync hasPty rescue closes the spawn/list race: a just-spawned PTY can // register after the inventory snapshot, and federation reads one // connected:false as exited. + let directLiveness: boolean | null | undefined + try { + directLiveness = leaf.ptyId ? this.ptyController?.hasPty?.(leaf.ptyId) : undefined + } catch { + directLiveness = null + } const provenAbsent = provenLivePtyIds !== null && leaf.ptyId !== null && !provenLivePtyIds.has(leaf.ptyId) && !leaf.ptyId.startsWith('remote:') && parseAppSshPtyId(leaf.ptyId) === null && - this.ptyController?.hasPty?.(leaf.ptyId) !== true + directLiveness !== true && + directLiveness !== null return { handle: this.issueHandle(leaf), ptyId: leaf.ptyId, @@ -37335,16 +37310,14 @@ function runtimePathsEqual(left: string, right: string): boolean { * Windows/WSL/SSH ids still match themselves across hosts. */ function runtimeWorktreeIdsEqual(left: string, right: string): boolean { - const parsedLeft = splitWorktreeId(left) - const parsedRight = splitWorktreeId(right) - return parsedLeft && parsedRight - ? parsedLeft.repoId === parsedRight.repoId && - runtimePathsEqual(parsedLeft.worktreePath, parsedRight.worktreePath) - : left === right + // Why: derived from the key rather than re-parsed, so equality and the sleep / + // mutation-queue keying can never drift apart into two different identity rules. + return runtimeWorktreeIdentityKey(left) === runtimeWorktreeIdentityKey(right) } function runtimeWorktreeIdentityKey(worktreeId: string): string { // Same suffix rule: this keys PTY refresh, sleep, and mutation-queue state per session. + // NUL cannot occur in a repoId or a path, so the joined key is unambiguous. const parsed = splitWorktreeId(worktreeId) return parsed ? `${parsed.repoId}\0${normalizeRuntimePathForComparison(parsed.worktreePath)}` diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index ec279aef763..a1ab618847c 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -39,7 +39,7 @@ import type { } from './types' import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display' import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-contract' -import { parsePaneKey } from '../../../shared/stable-pane-id' +import { isEquivalentPaneKey } from '../../../shared/stable-pane-id' import { OrchestrationError } from './orchestration-error' import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew' import { @@ -63,16 +63,6 @@ import { migrateMutationReceiptCapacity } from './mutation-receipt-capacity' -// Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. -function isEquivalentPaneKey(a: string, b: string): boolean { - if (a === b) { - return true - } - const aLeaf = parsePaneKey(a)?.leafId - const bLeaf = parsePaneKey(b)?.leafId - return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) -} - function parseWorkerTerminalPriorOwnerIds(value: string): string[] | null { try { const parsed: unknown = JSON.parse(value) diff --git a/src/main/runtime/orchestration/lifecycle-reconciliation.ts b/src/main/runtime/orchestration/lifecycle-reconciliation.ts index 3d2793413b7..7e9c2bc0efa 100644 --- a/src/main/runtime/orchestration/lifecycle-reconciliation.ts +++ b/src/main/runtime/orchestration/lifecycle-reconciliation.ts @@ -1,17 +1,6 @@ import type { OrchestrationDb } from './db' import type { MessageRow, WorkerReportOutcome } from './types' -import { parsePaneKey } from '../../../shared/stable-pane-id' - -// Why: the tab half can change on pane break-out, while opaque legacy keys -// have no safe equivalence beyond exact equality. -function isSamePane(assigneePaneKey: string, senderPaneKey: string): boolean { - if (assigneePaneKey === senderPaneKey) { - return true - } - const assigneeLeaf = parsePaneKey(assigneePaneKey)?.leafId - const senderLeaf = parsePaneKey(senderPaneKey)?.leafId - return Boolean(assigneeLeaf && senderLeaf && assigneeLeaf === senderLeaf) -} +import { isEquivalentPaneKey } from '../../../shared/stable-pane-id' function hasLifecycleAuthority( dispatch: { assignee_handle: string | null; assignee_pane_key: string | null }, @@ -19,7 +8,7 @@ function hasLifecycleAuthority( ): boolean { if (dispatch.assignee_pane_key) { return Boolean( - msg.sender_pane_key && isSamePane(dispatch.assignee_pane_key, msg.sender_pane_key) + msg.sender_pane_key && isEquivalentPaneKey(dispatch.assignee_pane_key, msg.sender_pane_key) ) } // Why: rows created before pane identity existed can only use the exact diff --git a/src/main/runtime/rpc/orchestration-legacy-process-identity.ts b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts index 35054c61279..14e4f3e46f3 100644 --- a/src/main/runtime/rpc/orchestration-legacy-process-identity.ts +++ b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts @@ -1,6 +1,8 @@ -import { parsePaneKey } from '../../../shared/stable-pane-id' +import { isEquivalentPaneKey } from '../../../shared/stable-pane-id' import { OrchestrationError } from '../orchestration/orchestration-error' +// Why: legacy rows carry nullable pane keys, and a missing key is never proof of +// identity — so absence is refused here rather than inside the shared comparison. export function equivalentLegacyPaneKey( a: string | null | undefined, b: string | null | undefined @@ -8,12 +10,7 @@ export function equivalentLegacyPaneKey( if (!a || !b) { return false } - if (a === b) { - return true - } - const aLeaf = parsePaneKey(a)?.leafId - const bLeaf = parsePaneKey(b)?.leafId - return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) + return isEquivalentPaneKey(a, b) } export function legacyReadOnlyError(): OrchestrationError { diff --git a/src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts b/src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts new file mode 100644 index 00000000000..5369e403195 --- /dev/null +++ b/src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts @@ -0,0 +1,157 @@ +/** + * Characterisation oracle for the 30-second SSH pane recovery grant in + * `OrcaRuntimeService.recoverTerminalPane`. + * + * The grant is the branch that, for a DISCONNECTED pane backed by a recently + * expired SSH lease, calls `createTerminal` to spawn a replacement shell. This + * file does not argue that the grant is wrong; it RECORDS that the branch is + * already dead at the id shapes production mints, so that deleting it is + * provably inert rather than assumed to be. Test 1 is written to survive that + * deletion unchanged: `terminal_not_recoverable` is the answer both before and + * after, and the anti-vacuity assertions pin that control genuinely reaches the + * grant gate instead of bailing out earlier. + * + * Why the branch is dead: the gate compares `pty.ptyId` (runtime, APP form — + * `SshPtyProvider` hands ids back through `toAppSshPtyId`, and that is the id + * `ipc/pty.ts` passes to `runtime.registerPty`) against `lease.ptyId` (durable, + * RELAY-native — `persistence.ts` normalizes every stored lease through + * `toRelaySshPtyId`). The comparison at `getRecentExpiredSshLease` is a raw + * `===` with no namespace normalization, so the two strings can never be equal. + * + * Honest limit on the claim. For an SSH pane the refusal is unreachable BY + * CONSTRUCTION: an app-form id always carries the `ssh:@@` prefix that + * the relay-native lease id, by definition, does not. For a LOCAL pane the + * argument is weaker — a local pty id is not namespaced, so the gate is + * unreachable only up to a random-UUID string collision with a stored lease id + * that also matches the worktree, tab and leaf. That is not a construction + * proof, just an astronomically unlikely one. Do not overclaim it. + */ +import { describe, expect, it, vi } from 'vitest' +import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' +import { OrcaRuntimeService } from './orca-runtime' + +const CONNECTION_ID = 'ssh-target-1' +const RELAY_PTY_ID = 'pty-7' +// Both id forms come from the production helpers so this oracle tracks the real +// namespace split rather than restating it as two hand-typed literals. +const APP_PTY_ID = toAppSshPtyId(CONNECTION_ID, RELAY_PTY_ID) +const STORED_LEASE_PTY_ID = toRelaySshPtyId(CONNECTION_ID, APP_PTY_ID) + +const REPO_ID = 'repo-1' +const WORKTREE_PATH = '/tmp/orca-grant-reachability' +const WORKTREE_ID = `${REPO_ID}::${WORKTREE_PATH}` +const TAB_ID = 'tab-grant' +// Persistence drops any leafId that is not a terminal leaf UUID, and the gate +// compares it, so the lease only qualifies with a real one. +const LEAF_ID = '11111111-1111-4111-8111-111111111111' +const PANE_KEY = `${TAB_ID}:${LEAF_ID}` + +function createStore() { + const now = Date.now() + return { + getRepo: (id: string) => createStore.repos.find((repo) => repo.id === id), + getRepos: () => createStore.repos, + getAllWorktreeMeta: () => ({ + [WORKTREE_ID]: { + displayName: 'grant', + comment: '', + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + linkedGitLabMR: null, + linkedGitLabIssue: null, + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + lastActivityAt: 0 + } + }), + getWorktreeMeta: () => undefined, + getSettings: () => ({ + workspaceDir: '/tmp/workspaces', + nestWorkspaces: false, + refreshLocalBaseRefOnWorktreeCreate: false, + branchPrefix: 'none', + branchPrefixCustom: '' + }), + getProjects: () => [], + getSparsePresets: () => [], + // Seeded to qualify on EVERY predicate of getRecentExpiredSshLease except + // the ptyId comparison: expired, same worktree/tab/leaf, inside the grace. + getSshRemotePtyLeases: () => [ + { + targetId: CONNECTION_ID, + ptyId: STORED_LEASE_PTY_ID, + worktreeId: WORKTREE_ID, + tabId: TAB_ID, + leafId: LEAF_ID, + state: 'expired' as const, + createdAt: now, + updatedAt: now + } + ] + } +} +createStore.repos = [ + { id: REPO_ID, path: WORKTREE_PATH, displayName: 'repo', badgeColor: 'blue', addedAt: 1 } +] + +/** Register one SSH-owned pty under `runtimePtyId`, then kill its shell. */ +function createDisconnectedPane(runtimePtyId: string) { + const runtime = new OrcaRuntimeService(createStore() as never) + runtime.registerPty(runtimePtyId, WORKTREE_ID, CONNECTION_ID, { + tabId: TAB_ID, + leafId: LEAF_ID + }) + const handle = runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).handle + runtime.onPtyExit(runtimePtyId, -1) + return { runtime, handle } +} + +describe('SSH pane recovery grant reachability', () => { + it('refuses a pane whose SSH shell died, at the id shapes production actually mints', async () => { + // The split itself, named: the durable lease id and the runtime pty id for + // the SAME shell are different strings. + expect(STORED_LEASE_PTY_ID).not.toBe(APP_PTY_ID) + + const { runtime, handle } = createDisconnectedPane(APP_PTY_ID) + const createTerminal = vi.spyOn(runtime, 'createTerminal') + + // Anti-vacuity: control must actually arrive at the grant gate. A truthy + // handle rules out an early `terminal_not_found`; connected === false rules + // out the still-connected branch that also throws not_recoverable. + expect(handle).toBeTruthy() + expect(runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).connected).toBe(false) + + await expect(runtime.recoverTerminalPane(PANE_KEY, WORKTREE_ID, handle)).rejects.toThrow( + 'terminal_not_recoverable' + ) + expect(createTerminal).not.toHaveBeenCalled() + }) + + // INVERTED BY THE DELETION, and that inversion is the point. + // + // Before the grant was removed, this case RESOLVED with a replacement: the same seeded lease, + // with the runtime pty registered under the bare relay id instead of the app-form one, opened + // the gate. That is what made the case above attributable to the namespace and to nothing else — + // the lease qualified on state, worktree, tab, leaf and grace window, and only the id comparison + // stood between it and a spawn. + // + // The shape was never reachable in production (a lease implies a truthy connectionId, which + // implies an app-form id), so flipping it to a refusal changes no real behaviour — which is + // exactly what "the deletion is inert" means. Kept rather than dropped so the counterfactual + // stays on record: no id shape authorizes a spawn now, not merely the one production mints. + it('refuses even the id shape that used to open the grant', async () => { + const { runtime, handle } = createDisconnectedPane(RELAY_PTY_ID) + const createTerminal = vi.spyOn(runtime, 'createTerminal') + + expect(handle).toBeTruthy() + expect(runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).connected).toBe(false) + + await expect(runtime.recoverTerminalPane(PANE_KEY, WORKTREE_ID, handle)).rejects.toThrow( + 'terminal_not_recoverable' + ) + expect(createTerminal).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts b/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts index 5cbb967da76..9ca0c021333 100644 --- a/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts +++ b/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts @@ -148,6 +148,23 @@ describe('listTerminals liveness truth for restored leaves', () => { }) }) + it('keeps a leaf connected when the snapshot missed it but provider liveness is unknown', async () => { + const runtime = makeRuntimeWithLeaf({ + leafPtyId: 'pty-provider-disconnected', + controllerSessions: [], + hasPty: () => null + }) + + const { terminals } = await runtime.listTerminals(`id:${WORKTREE_ID}`) + + expect(terminals).toHaveLength(1) + expect(terminals[0]).toMatchObject({ + ptyId: 'pty-provider-disconnected', + connected: true, + writable: true + }) + }) + it('does not demote remote-runtime-scoped leaves the local inventory never covers', async () => { const runtime = makeRuntimeWithLeaf({ leafPtyId: 'remote:env-1@@term_abc', diff --git a/src/main/runtime/workspace-namespace-terminal-identity.test.ts b/src/main/runtime/workspace-namespace-terminal-identity.test.ts new file mode 100644 index 00000000000..d3e1a3d19bc --- /dev/null +++ b/src/main/runtime/workspace-namespace-terminal-identity.test.ts @@ -0,0 +1,365 @@ +import { describe, expect, it } from 'vitest' +import { FLOATING_TERMINAL_WORKTREE_ID, getDefaultWorkspaceSession } from '../../shared/constants' +import { makePaneKey } from '../../shared/stable-pane-id' +import type { WorkspaceSessionState } from '../../shared/types' +import { FOLDER_WORKSPACE_INSTANCE_SEPARATOR } from '../../shared/worktree-id' +import { OrcaRuntimeService } from './orca-runtime' + +/** + * Journey: git-worktree, folder-workspace and floating namespaces must all resolve + * terminal identity from one stable host-local workspace id. Neither the filesystem + * path, the client's repository id, nor the caller's target id may stand in for it. + * Drive-letter and UNC ids normalize host-independently and are covered here; the + * Windows PTY lifecycle behind them is not. + */ + +const GIT_REPO_ID = 'repo-git' +const GIT_ROOT_PATH = '/tmp/ns/checkout' +const GIT_ROOT_ID = `${GIT_REPO_ID}::${GIT_ROOT_PATH}` +const GIT_CHILD_PATH = `${GIT_ROOT_PATH}/.claude/worktrees/child` +const GIT_CHILD_ID = `${GIT_REPO_ID}::${GIT_CHILD_PATH}` + +const FOLDER_REPO_ID = 'repo-folder' +const FOLDER_PATH = '/tmp/ns/folder' +const FOLDER_ROOT_ID = `${FOLDER_REPO_ID}::${FOLDER_PATH}` +const FOLDER_A = `${FOLDER_ROOT_ID}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa` +const FOLDER_B = `${FOLDER_ROOT_ID}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb` + +// Same directory, re-added as a second project: only the repo id differs. +const ALIAS_REPO_ID = 'repo-folder-alias' +const ALIAS_ID = `${ALIAS_REPO_ID}::${FOLDER_PATH}` + +const PTY_GIT_ROOT = `${GIT_ROOT_ID}@@pty-git-root` +const PTY_GIT_CHILD = `${GIT_CHILD_ID}@@pty-git-child` +const PTY_A = `${FOLDER_A}@@pty-a` +const PTY_B = `${FOLDER_B}@@pty-b` +const PTY_FOLDER_ROOT = `${FOLDER_ROOT_ID}@@pty-folder-root` +const PTY_ALIAS = `${ALIAS_ID}@@pty-alias` +const PTY_FLOATING = 'floating-pty' + +const LEAF_A = 'a1111111-1111-4111-8111-111111111111' +const LEAF_B = 'b2222222-2222-4222-8222-222222222222' + +const REPOS = [ + { id: GIT_REPO_ID, path: GIT_ROOT_PATH, displayName: 'checkout', badgeColor: 'blue', addedAt: 1 }, + { + id: FOLDER_REPO_ID, + path: FOLDER_PATH, + displayName: 'folder', + badgeColor: 'green', + addedAt: 2, + kind: 'folder' + }, + { + id: ALIAS_REPO_ID, + path: FOLDER_PATH, + displayName: 'folder-alias', + badgeColor: 'red', + addedAt: 3, + kind: 'folder' + } +] as const + +const ALL_WORKSPACE_IDS = [GIT_ROOT_ID, GIT_CHILD_ID, FOLDER_ROOT_ID, FOLDER_A, FOLDER_B, ALIAS_ID] + +type ControllerSession = { + id: string + worktreeId?: string + cwd?: string + title?: string + incarnationId?: string + terminalHandle?: string +} + +type RuntimeInternals = { + buildResolvedWorktreeFromId: (worktreeId: string) => unknown + refreshPtyWorktreeRecordsWithControllerInventory: ( + resolvedWorktrees: unknown[], + targetWorktreeId?: string | null + ) => Promise<{ livePtyIds: Set } | null> + getLivePtyIdsForWorktree: (worktreeId: string) => Set + hasExactPersistedTerminalSurfaceIdentity: (expected: { + worktreeId: string + tabId: string + leafId: string + ptyId: string + incarnationId: string + }) => boolean + ptysById: Map + recordPtyWorktree: (ptyId: string, worktreeId: string, state?: Record) => unknown +} + +function createRuntimeInternals(options: { + session?: WorkspaceSessionState + sessions?: ControllerSession[] +}): RuntimeInternals { + const meta = Object.fromEntries(ALL_WORKSPACE_IDS.map((id) => [id, { hostId: 'local' }])) + const store = { + getRepos: () => REPOS, + getRepo: (id: string) => REPOS.find((repo) => repo.id === id), + getAllWorktreeMeta: () => meta, + getWorktreeMeta: (worktreeId: string) => meta[worktreeId], + setWorktreeMeta: (worktreeId: string, patch: Record) => { + meta[worktreeId] = { ...meta[worktreeId], ...patch } + return meta[worktreeId] + }, + getWorkspaceSession: () => options.session ?? getDefaultWorkspaceSession(), + setWorkspaceSession: () => {}, + flushOrThrow: () => {} + } as never + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + stopAndWait: async () => true, + getForegroundProcess: async () => null, + listProcesses: async () => options.sessions ?? [] + } as never) + return runtime as unknown as RuntimeInternals +} + +function resolveWorktrees(internals: RuntimeInternals, ids: readonly string[]): unknown[] { + return ids.map((id) => internals.buildResolvedWorktreeFromId(id)) +} + +/** Persisted layout binding each folder sibling's own PTY to its own tab and leaf. */ +function persistedFolderSiblingSession(): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [FOLDER_A]: [{ id: 'tab-a', worktreeId: FOLDER_A, title: 'A' }], + [FOLDER_B]: [{ id: 'tab-b', worktreeId: FOLDER_B, title: 'B' }] + } as never, + terminalLayoutsByTabId: { + 'tab-a': { root: null, activeLeafId: LEAF_A, ptyIdsByLeafId: { [LEAF_A]: PTY_A } }, + 'tab-b': { root: null, activeLeafId: LEAF_B, ptyIdsByLeafId: { [LEAF_B]: PTY_B } } + } as never, + terminalPtyIncarnationsByPaneKey: { + [makePaneKey('tab-a', LEAF_A)]: 'inc-a', + [makePaneKey('tab-b', LEAF_B)]: 'inc-b' + } + } +} + +describe('workspace namespaces keep terminal identity host-local and per-workspace', () => { + describe('git worktree namespace', () => { + it('binds a nested worktree PTY to the nested worktree, not the targeted parent', async () => { + // cwd alone cannot separate a parent checkout from a worktree nested inside it. + const internals = createRuntimeInternals({ + sessions: [{ id: 'unattributed-pty', cwd: `${GIT_CHILD_PATH}/src`, title: 'child work' }] + }) + const resolved = resolveWorktrees(internals, [GIT_ROOT_ID, GIT_CHILD_ID]) + + const scopedToParent = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + GIT_ROOT_ID + ) + const scopedToChild = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + GIT_CHILD_ID + ) + + expect([...(scopedToParent?.livePtyIds ?? [])]).toEqual([]) + expect([...(scopedToChild?.livePtyIds ?? [])]).toEqual(['unattributed-pty']) + expect(internals.ptysById.get('unattributed-pty')?.worktreeId).toBe(GIT_CHILD_ID) + }) + + it('keeps a checkout and the worktree nested inside it in separate namespaces', async () => { + const internals = createRuntimeInternals({ + sessions: [ + { id: PTY_GIT_ROOT, worktreeId: GIT_ROOT_ID, cwd: GIT_ROOT_PATH, title: 'git' }, + { id: PTY_GIT_CHILD, worktreeId: GIT_CHILD_ID, cwd: GIT_CHILD_PATH, title: 'child' } + ] + }) + const resolved = resolveWorktrees(internals, [GIT_ROOT_ID, GIT_CHILD_ID]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + + expect(internals.ptysById.get(PTY_GIT_ROOT)?.worktreeId).toBe(GIT_ROOT_ID) + expect(internals.ptysById.get(PTY_GIT_CHILD)?.worktreeId).toBe(GIT_CHILD_ID) + expect([...internals.getLivePtyIdsForWorktree(GIT_ROOT_ID)]).toEqual([PTY_GIT_ROOT]) + expect([...internals.getLivePtyIdsForWorktree(GIT_CHILD_ID)]).toEqual([PTY_GIT_CHILD]) + }) + }) + + describe('folder workspace namespace', () => { + it('does not let the caller target stand in for an attributed PTY owner', async () => { + const internals = createRuntimeInternals({ + sessions: [{ id: PTY_B, worktreeId: FOLDER_B, cwd: FOLDER_PATH, title: 'b' }] + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + const scopedToA = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_A + ) + + expect([...(scopedToA?.livePtyIds ?? [])]).toEqual([]) + expect(internals.ptysById.get(PTY_B)?.worktreeId).toBe(FOLDER_B) + }) + + it('keeps each sibling terminal bound to its own workspace across a reconnect', async () => { + const sessions: ControllerSession[] = [ + { + id: PTY_A, + worktreeId: FOLDER_A, + cwd: FOLDER_PATH, + title: 'a', + incarnationId: 'inc-a', + terminalHandle: 'term_a' + }, + { + id: PTY_B, + worktreeId: FOLDER_B, + cwd: FOLDER_PATH, + title: 'b', + incarnationId: 'inc-b', + terminalHandle: 'term_b' + } + ] + const internals = createRuntimeInternals({ + session: persistedFolderSiblingSession(), + sessions + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B]) + + // Reconnect resync: each workspace re-lists the same controller inventory. + const afterA = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_A + ) + const afterB = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_B + ) + + expect([...(afterA?.livePtyIds ?? [])]).toEqual([PTY_A]) + expect([...(afterB?.livePtyIds ?? [])]).toEqual([PTY_B]) + expect(internals.ptysById.get(PTY_A)?.worktreeId).toBe(FOLDER_A) + expect(internals.ptysById.get(PTY_B)?.worktreeId).toBe(FOLDER_B) + expect( + internals.hasExactPersistedTerminalSurfaceIdentity({ + worktreeId: FOLDER_B, + tabId: 'tab-b', + leafId: LEAF_B, + ptyId: PTY_B, + incarnationId: 'inc-b' + }) + ).toBe(true) + // A sibling must never satisfy the other's persisted surface identity. + expect( + internals.hasExactPersistedTerminalSurfaceIdentity({ + worktreeId: FOLDER_A, + tabId: 'tab-b', + leafId: LEAF_B, + ptyId: PTY_B, + incarnationId: 'inc-b' + }) + ).toBe(false) + }) + + it('does not treat one directory as one namespace across two projects', async () => { + // Same directory added twice: the ids differ only by the client's repo id. + const internals = createRuntimeInternals({ + sessions: [ + { id: PTY_A, worktreeId: FOLDER_A, cwd: FOLDER_PATH, title: 'a' }, + { id: PTY_FOLDER_ROOT, worktreeId: FOLDER_ROOT_ID, cwd: FOLDER_PATH, title: 'root' }, + { id: PTY_ALIAS, worktreeId: ALIAS_ID, cwd: FOLDER_PATH, title: 'alias' } + ] + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B, FOLDER_ROOT_ID, ALIAS_ID]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + + expect(internals.ptysById.get(PTY_A)?.worktreeId).toBe(FOLDER_A) + expect(internals.ptysById.get(PTY_FOLDER_ROOT)?.worktreeId).toBe(FOLDER_ROOT_ID) + expect(internals.ptysById.get(PTY_ALIAS)?.worktreeId).toBe(ALIAS_ID) + expect([...internals.getLivePtyIdsForWorktree(ALIAS_ID)]).toEqual([PTY_ALIAS]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_ROOT_ID)]).toEqual([PTY_FOLDER_ROOT]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_A)]).toEqual([PTY_A]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_B)]).toEqual([]) + }) + }) + + describe('floating workspace namespace', () => { + it('never lets a repo workspace claim a floating terminal or vice versa', async () => { + const internals = createRuntimeInternals({ + sessions: [ + { + id: PTY_FLOATING, + worktreeId: FLOATING_TERMINAL_WORKTREE_ID, + cwd: FOLDER_PATH, + title: 'floating' + }, + { id: PTY_A, worktreeId: FOLDER_A, cwd: FOLDER_PATH, title: 'a' } + ] + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + const scopedToA = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_A + ) + + // The floating id carries no repo/path pair, so it can only equal itself. + expect([...(scopedToA?.livePtyIds ?? [])]).toEqual([PTY_A]) + expect(internals.ptysById.get(PTY_FLOATING)?.worktreeId).toBe(FLOATING_TERMINAL_WORKTREE_ID) + expect([...internals.getLivePtyIdsForWorktree(FLOATING_TERMINAL_WORKTREE_ID)]).toEqual([ + PTY_FLOATING + ]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_A)]).toEqual([PTY_A]) + }) + + it('does not resolve a floating terminal onto a workspace that shares its cwd', () => { + const internals = createRuntimeInternals({}) + internals.recordPtyWorktree(PTY_FLOATING, FLOATING_TERMINAL_WORKTREE_ID, { connected: true }) + internals.recordPtyWorktree(PTY_A, FOLDER_A, { connected: true }) + + expect([...internals.getLivePtyIdsForWorktree(FOLDER_ROOT_ID)]).toEqual([]) + expect([...internals.getLivePtyIdsForWorktree(FLOATING_TERMINAL_WORKTREE_ID)]).toEqual([ + PTY_FLOATING + ]) + }) + }) + + // Only the id-normalization half of the Windows namespaces is host-independent. + // Spawning and reconnecting a real drive/UNC PTY still needs a Windows host. + describe('drive-letter and UNC namespaces (id normalization only)', () => { + it('folds drive spelling and WSL UNC aliases but keeps distinct roots apart', () => { + const internals = createRuntimeInternals({}) + internals.recordPtyWorktree('pty-c', 'repo-win::C:\\ws\\proj', { connected: true }) + internals.recordPtyWorktree('pty-d', 'repo-win::D:\\ws\\proj', { connected: true }) + internals.recordPtyWorktree('pty-share', 'repo-win::\\\\server\\share\\proj', { + connected: true + }) + internals.recordPtyWorktree('pty-other-share', 'repo-win::\\\\server\\other\\proj', { + connected: true + }) + internals.recordPtyWorktree('pty-wsl', 'repo-win::\\\\wsl$\\Ubuntu\\home\\me', { + connected: true + }) + + expect([...internals.getLivePtyIdsForWorktree('repo-win::c:/ws/proj')]).toEqual(['pty-c']) + expect([...internals.getLivePtyIdsForWorktree('repo-win::D:/ws/proj')]).toEqual(['pty-d']) + expect([...internals.getLivePtyIdsForWorktree('repo-win::\\\\server\\share\\proj')]).toEqual([ + 'pty-share' + ]) + expect([ + ...internals.getLivePtyIdsForWorktree('repo-win::\\\\wsl.localhost\\ubuntu\\home\\me') + ]).toEqual(['pty-wsl']) + }) + + it('keeps a folder workspace instance on a drive path distinct from its root', () => { + const driveRoot = 'repo-win::C:\\ws\\proj' + const driveInstance = `${driveRoot}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}dddddddd-dddd-4ddd-8ddd-dddddddddddd` + const internals = createRuntimeInternals({}) + internals.recordPtyWorktree('pty-drive-root', driveRoot, { connected: true }) + internals.recordPtyWorktree('pty-drive-instance', driveInstance, { connected: true }) + + expect([...internals.getLivePtyIdsForWorktree(driveRoot)]).toEqual(['pty-drive-root']) + expect([...internals.getLivePtyIdsForWorktree(driveInstance)]).toEqual(['pty-drive-instance']) + }) + }) +}) diff --git a/src/main/runtime/workspace-session-terminal-membership-authority.ts b/src/main/runtime/workspace-session-terminal-membership-authority.ts index 9dace4c8cb1..9509f14eb26 100644 --- a/src/main/runtime/workspace-session-terminal-membership-authority.ts +++ b/src/main/runtime/workspace-session-terminal-membership-authority.ts @@ -161,6 +161,30 @@ export function advanceTerminalTopologyRevision( } } +/** + * The tab whose live layout holds this leaf. Only the leaf half of a pane key is stable — breaking + * a pane out into its own tab moves the leaf and leaves any stored tabId naming the tab it left. + */ +export function findTerminalTabIdForLeaf( + session: WorkspaceSessionState | undefined, + leafId: string +): string | undefined { + // Only a still-existing tab counts: a layout entry outlives its tab, and binding a live shell to + // a deleted one registers a pane under a ghost. Scanned across worktrees rather than under one + // key so the answer cannot hinge on a worktreeId matching exactly (`::workspace:` suffixes). + const liveTabIds = new Set( + Object.values(session?.tabsByWorktree ?? {}).flatMap((tabs) => tabs.map((tab) => tab.id)) + ) + for (const [tabId, layout] of Object.entries(session?.terminalLayoutsByTabId ?? {})) { + const leafIds = new Set() + collectLeafIds(layout.root, leafIds) + if (liveTabIds.has(tabId) && leafIds.has(leafId)) { + return tabId + } + } + return undefined +} + export function hasHostAuthoritativeTerminalMembership( session: WorkspaceSessionState | undefined, worktreeId: string diff --git a/src/main/ssh-pane-binding-partition.test.ts b/src/main/ssh-pane-binding-partition.test.ts new file mode 100644 index 00000000000..a630f62a941 --- /dev/null +++ b/src/main/ssh-pane-binding-partition.test.ts @@ -0,0 +1,399 @@ +/** + * STA-3077 step P: an SSH pane's durable binding must live in ONE partition. + * + * Today it lives in two. Main's spawn writes `ssh:` + * (ipc/pty.ts persistPtyBinding(binding, toSshExecutionHostId(connectionId))), + * the relay's reattach write passes no hostId and lands in `local` + * (ssh-relay-session.ts restoreReattachedPtyRuntime), and the renderer keeps SSH + * worktrees in `local` on purpose (buildHostIdByWorktreeId). Supersession then + * reads a binding no live writer maintains, so it compares the arriving lease + * against a stale pty id, bails, and leaves the predecessor live. That is the + * reported 2 -> 19 -> 20 mechanism. + * + * These oracles assert cardinality and identity — one pane, one live claim, and + * the surviving claim is the shell the pane is bound to — never which function + * ran, so they stay valid under any single-accessor implementation. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { getDefaultPersistedState, getDefaultWorkspaceSession } from '../shared/constants' +import { LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId } from '../shared/execution-host' +import { toAppSshPtyId, toRelaySshPtyId } from '../shared/ssh-pty-id' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('./telemetry/client', () => ({ track: vi.fn() })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TARGET = 'ssh-target-1' +const SSH_PARTITION = toSshExecutionHostId(TARGET) +const WORKTREE = 'repo-1:wt-1' +const TAB = 'tab-1' +/** Must be a real layout leaf UUID — the store drops any other spelling. */ +const LEAF = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + +const appPtyId = (relayPtyId: string): string => toAppSshPtyId(TARGET, relayPtyId) + +async function createStore(state: Record = {}) { + mkdirSync(testState.dir, { recursive: true }) + writeFileSync( + join(testState.dir, 'orca-data.json'), + JSON.stringify({ ...getDefaultPersistedState(testState.dir), ...state }), + 'utf-8' + ) + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() +} + +type TestStore = Awaited> + +/** One SSH pane, bound to `relayPtyId`, as a partition stores it. */ +function paneSession(relayPtyId: string) { + const ptyId = appPtyId(relayPtyId) + return { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [WORKTREE]: [ + { + id: TAB, + ptyId, + worktreeId: WORKTREE, + title: 'Terminal 1', + defaultTitle: 'Terminal 1', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + [TAB]: { + root: { type: 'leaf' as const, leafId: LEAF }, + activeLeafId: LEAF, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF]: ptyId } + } + } + } +} + +/** session:set for an SSH worktree — buildHostIdByWorktreeId sends it to `local`. */ +function rendererPublishesPane(store: TestStore, relayPtyId: string): void { + store.setWorkspaceSession(paneSession(relayPtyId) as never, LOCAL_EXECUTION_HOST_ID) +} + +/** A mid-session write into `ssh:` — orphan adoption still targets that partition, so the + * one-time load fold cannot be the only thing keeping the two homes from reappearing. */ +function somethingRewritesTheSshPartition(store: TestStore, relayPtyId: string): void { + store.setWorkspaceSession(paneSession(relayPtyId) as never, SSH_PARTITION) +} + +/** ssh-relay-session.ts:2504 — the reattach bind. No hostId, refuses to create. */ +function relayReattachBindsPane(store: TestStore, relayPtyId: string): boolean | null { + return store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: appPtyId(relayPtyId), + incarnationId: `inc-${relayPtyId}`, + mayCreate: false + }) +} + +/** ipc/pty.ts:6493 — the spawn's lease upsert, ahead of its binding write. */ +function sshSpawnUpsertsLease(store: TestStore, relayPtyId: string): void { + store.upsertSshRemotePtyLease({ + targetId: TARGET, + ptyId: toRelaySshPtyId(TARGET, appPtyId(relayPtyId)), + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + state: 'attached', + lastAttachedAt: Date.now() + }) +} + +function liveLeaseIdsForPane(store: TestStore): string[] { + return store + .getSshRemotePtyLeases(TARGET) + .filter( + (lease) => + lease.tabId === TAB && + lease.leafId === LEAF && + lease.state !== 'terminated' && + lease.state !== 'expired' + ) + .map((lease) => lease.ptyId) + .sort() +} + +/** The argument text of every `callee(...)` call in a production source file. */ +function callArgumentsIn(source: string, callee: string): string[] { + const calls: string[] = [] + for (let index = source.indexOf(`${callee}(`); index !== -1; ) { + let cursor = index + callee.length + const start = cursor + 1 + for (let depth = 0; cursor < source.length; cursor += 1) { + if (source[cursor] === '(') { + depth += 1 + } else if (source[cursor] === ')' && --depth === 0) { + break + } + } + calls.push(source.slice(start, cursor)) + index = source.indexOf(`${callee}(`, cursor) + } + return calls +} + +/** On-disk state from an earlier session: both partitions name the same shell, + * because main's spawn wrote `ssh:` and the renderer published `local`. + * They diverge as soon as the relay's reattach write updates only `local`. */ +function diskAfterEarlierSession(relayPtyId: string) { + return { + workspaceSession: paneSession(relayPtyId), + workspaceSessionsByHostId: { [SSH_PARTITION]: paneSession(relayPtyId) } + } +} + +beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-sta3077-partition-')) +}) + +describe('STA-3077 step P: one pane, one live claim across partitions', () => { + // The pane is bound to pty-2 in `local` (the relay's reattach write put it + // there); the SSH partition still names the predecessor pty-1. Supersession + // reads ssh-first, sees pty-1 != pty-2, and bails — both shells stay claimed. + it('supersedes the predecessor when the pane is bound in the local partition', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + sshSpawnUpsertsLease(store, 'pty-1') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + + sshSpawnUpsertsLease(store, 'pty-2') + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) + + // reattachKnownPtys calls this first, so a wrong winner here is what fans the + // reconnect out over a dead id and grafts a pane the user never opened. + it('keeps the lease the live pane binding names when healing duplicates', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + sshSpawnUpsertsLease(store, 'pty-1') + sshSpawnUpsertsLease(store, 'pty-2') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + + await store.supersedeDuplicatePaneLeases(TARGET) + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) + + // Isolates the reader from the one-time load fold. Without this clause the fold masks the + // partition preference — it deletes the divergent copy at boot, so restoring the ssh-first + // hedge stays green and the reader guard ships unproven. Anything that writes `ssh:` + // after load (orphan adoption still does) would then revive the defect inside one session. + it('supersedes the predecessor when the ssh partition is rewritten mid-session', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + sshSpawnUpsertsLease(store, 'pty-1') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + somethingRewritesTheSshPartition(store, 'pty-1') + + sshSpawnUpsertsLease(store, 'pty-2') + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) + + // The reported growth: live claims must not scale with reconnect count. + it('holds the live claim count flat across ten reconnects of one pane', async () => { + const store = await createStore(diskAfterEarlierSession('pty-0')) + sshSpawnUpsertsLease(store, 'pty-0') + + for (let reconnect = 1; reconnect <= 10; reconnect += 1) { + expect(relayReattachBindsPane(store, `pty-${reconnect}`)).toBe(true) + sshSpawnUpsertsLease(store, `pty-${reconnect}`) + } + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-10']) + }) +}) + +describe('STA-3077: a decision made from one plane only mutates that plane', () => { + // Arbitration ranks using the desktop plane's binding. The headless plane's binding never gets a + // vote, so deleting it here would strand a shell its owner may still be using — and that owner + // would then have no durable record to reattach by. An explicit expiry is different: the pty is + // gone for everyone, and that path still scrubs both. + it('leaves the headless plane binding intact when supersession retires its lease', async () => { + const sshSession = paneSession('pty-old') as unknown as Record + const store = await createStore({ + workspaceSession: paneSession('pty-new'), + workspaceSessionsByHostId: { [SSH_PARTITION]: sshSession } + }) + sshSpawnUpsertsLease(store, 'pty-old') + + // Local names pty-new, so arbitration expires pty-old without the other plane being consulted. + sshSpawnUpsertsLease(store, 'pty-new') + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-new']) + expect( + store.getWorkspaceSession(SSH_PARTITION).terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF] + ).toBe(appPtyId('pty-old')) + }) +}) + +describe('STA-3077: a pane moved between tabs still supersedes its own predecessor', () => { + // The reported cardinality growth, in its surviving form. A lease freezes its tabId at write + // time. Break the pane out into a new tab and its next lease carries the NEW tab, so matching + // siblings on tabId means the two leases for one pane never meet — the predecessor is never + // superseded and the count grows on every reconnect, exactly as reported. + function leaseInTab(store: TestStore, relayPtyId: string, tabId: string): void { + store.upsertSshRemotePtyLease({ + targetId: TARGET, + ptyId: toRelaySshPtyId(TARGET, appPtyId(relayPtyId)), + worktreeId: WORKTREE, + tabId, + leafId: LEAF, + state: 'attached', + lastAttachedAt: Date.now() + }) + } + const liveForLeaf = (store: TestStore): string[] => + store + .getSshRemotePtyLeases(TARGET) + .filter((l) => l.leafId === LEAF && l.state !== 'terminated' && l.state !== 'expired') + .map((l) => l.ptyId) + .sort() + + it('supersedes a predecessor whose lease names the tab the pane left', async () => { + const store = await createStore() + leaseInTab(store, 'pty-old', TAB) + + leaseInTab(store, 'pty-new', 'tab-moved-to') + + expect(liveForLeaf(store)).toEqual(['pty-new']) + }) + + it('holds the count flat across ten reconnects that each land in a new tab', async () => { + const store = await createStore() + leaseInTab(store, 'pty-0', TAB) + for (let n = 1; n <= 10; n += 1) { + leaseInTab(store, `pty-${n}`, `tab-${n}`) + } + + expect(liveForLeaf(store)).toEqual(['pty-10']) + }) +}) + +describe('STA-3077: arbitration follows the pane, not the tab it was written in', () => { + // A lease freezes its tabId when written; `detachTerminalPaneToTab` then moves the live pane and + // its PTY into a new tab. Looking the binding up under the frozen tab finds nothing, so the bound + // shell loses to recency and supersession retires the pane's OWN shell in favour of a stale one. + it('finds the binding after the pane has been moved to another tab', async () => { + const MOVED_TAB = 'tab-moved-to' + const store = await createStore() + // The renderer publishes the pane in its NEW tab; the lease still names the old one. + store.setWorkspaceSession( + { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [WORKTREE]: [{ id: MOVED_TAB, worktreeId: WORKTREE }] }, + terminalLayoutsByTabId: { + [MOVED_TAB]: { + root: { type: 'leaf' as const, leafId: LEAF }, + activeLeafId: LEAF, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF]: appPtyId('pty-bound') } + } + } + } as never, + LOCAL_EXECUTION_HOST_ID + ) + sshSpawnUpsertsLease(store, 'pty-bound') + // A newer, unbound lease arrives for the same pane under the lease's frozen tab. + sshSpawnUpsertsLease(store, 'pty-newer') + + await store.supersedeDuplicatePaneLeases(TARGET) + + // The bound shell must win. Keyed on the frozen tab, it is invisible and recency retires it. + expect(liveLeaseIdsForPane(store)).toEqual(['pty-bound']) + }) +}) + +describe('STA-3077 step P: the desktop plane resolves from its own home', () => { + // INVERTED. This clause used to require the two partitions to AGREE after load, which assumed + // `ssh:` was a stale spill of this plane's state. It is not — it is the headless/CLI + // plane's own home, which that plane writes and reads deliberately (STA-3463, STA-3465). The + // property that actually matters is narrower and stronger: what the desktop plane resolves must + // follow `local` alone, whatever the other plane holds. Asserting agreement made a migration look + // necessary that in fact erased another plane's live state. + it('resolves to the local binding regardless of what the ssh partition holds', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + + rendererPublishesPane(store, 'pty-1') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + // The other plane names the predecessor. It does not get a vote here — local speaking about + // this leaf IS the desktop plane's live view, and letting the other copy outrank it is the + // STA-3077 defect. Its binding to the loser is then scrubbed with every other reference to + // that retired lease, so the two planes do not drift further apart. + somethingRewritesTheSshPartition(store, 'pty-1') + sshSpawnUpsertsLease(store, 'pty-2') + + expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe( + appPtyId('pty-2') + ) + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) +}) + +describe('STA-3077 step P: every production caller names that one home', () => { + const summarize = (source: string): string => source.replace(/\s+/g, ' ').trim().slice(0, 90) + + // A guard that behaves correctly is not evidence that both writers reach it, + // and the defect IS that they disagree — so pin the call sites too. + it('has no SSH pane binding write that selects the ssh partition', () => { + const file = 'src/main/ipc/pty.ts' + const calls = callArgumentsIn(readFileSync(file, 'utf-8'), 'persistPtyBinding') + expect(calls.length, `${file} no longer writes pane bindings`).toBeGreaterThan(0) + + const partitioned = calls + .filter((call) => call.includes('toSshExecutionHostId')) + .map((call) => `${file}: ${summarize(call)}`) + + expect(partitioned).toEqual([]) + }) + + // STRENGTHENED, not relaxed. This clause used to require the relay to hold a `persistPtyBinding` + // call of its own and merely forbid an ssh-partition argument on it. Step F removed that call: + // the relay binds through the one `bindPaneShell` producer, which is what makes the superseded- + // pane fence live on reattach. Requiring ZERO direct writes here is the stronger property — a + // second bind producer is exactly the defect that let spawn and reattach disagree. + it('has no pane binding write in the relay that bypasses the one bind producer', () => { + const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + + expect(callArgumentsIn(source, 'persistPtyBinding').map(summarize)).toEqual([]) + expect(source).toContain('bindPaneShell(') + }) + + // The readers must land in the same place; one that still consults + // `ssh:` reinstates the disagreement from the other side. + it('has no stable-pane owner reader that selects the ssh partition', () => { + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + const start = source.indexOf('function resolvePersistedStablePaneOwner') + const readers = source.slice(start, source.indexOf('type StablePaneSpawnContext')) + expect(start, 'stable-pane owner readers moved').toBeGreaterThan(0) + expect(readers).toContain('getWorkspaceSession(') + + const partitioned = readers + .split('\n') + .filter((line) => line.includes('toSshExecutionHostId')) + .map(summarize) + + expect(partitioned).toEqual([]) + }) +}) diff --git a/src/main/ssh-reattach-pane-cardinality.test.ts b/src/main/ssh-reattach-pane-cardinality.test.ts new file mode 100644 index 00000000000..8f5665981c4 --- /dev/null +++ b/src/main/ssh-reattach-pane-cardinality.test.ts @@ -0,0 +1,742 @@ +/** + * STA-3077 behavioral oracles: reconnecting an SSH workspace must not add panes + * the user never opened, and must not accumulate remote shells. + * + * These assert observable behavior, not a mechanism, so they stay valid under + * any implementation that fixes the defect. Each case names the root cause it + * pins from the #12264 diagnosis. + * + * Reported symptom: relay PTY count went 2 -> 19 -> 20 across three reconnects. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { getDefaultPersistedState } from '../shared/constants' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('./telemetry/client', () => ({ track: vi.fn() })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TARGET = 'ssh-target-1' +const WORKTREE = 'repo-1:wt-1' +const TAB = 'tab-1' +/** Must be a real layout leaf UUID — the store drops any other spelling. */ +const LEAF = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + +async function createStore(state: Record = {}) { + mkdirSync(testState.dir, { recursive: true }) + writeFileSync( + join(testState.dir, 'orca-data.json'), + JSON.stringify({ ...getDefaultPersistedState(testState.dir), ...state }), + 'utf-8' + ) + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() +} + +beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-sta3077-')) +}) + +/** One pane's lease, as the reattach path records it. */ +function leaseFor(ptyId: string, updatedAt: number) { + return { + targetId: TARGET, + ptyId, + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + state: 'attached' as const, + updatedAt + } +} + +function liveLeasesForPane(store: { + getSshRemotePtyLeases: (targetId?: string) => readonly { + ptyId: string + tabId?: string + leafId?: string + state: string + }[] +}) { + return store + .getSshRemotePtyLeases(TARGET) + .filter( + (lease) => + lease.tabId === TAB && + lease.leafId === LEAF && + lease.state !== 'terminated' && + lease.state !== 'expired' + ) +} + +describe('STA-3077: one pane owns at most one live remote PTY lease', () => { + // RC1: lease uniqueness keys on (targetId, ptyId) only, so a pane whose PTY id + // changes leaves its predecessor behind with nothing to retire it. + it('does not accumulate a second live lease when one pane re-leases a new PTY id', async () => { + const store = await createStore() + + store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1)) + store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2)) + + expect(liveLeasesForPane(store)).toHaveLength(1) + }) + + // RC1: the reported 2 -> 19 -> 20 growth. Lease count must not scale with + // reconnect count for a fixed set of panes. + it('keeps live lease count flat across repeated reconnects of one pane', async () => { + const store = await createStore() + + for (let reconnect = 0; reconnect < 10; reconnect += 1) { + store.upsertSshRemotePtyLease(leaseFor(`relay-pty-${reconnect}`, reconnect + 1)) + } + + expect(liveLeasesForPane(store)).toHaveLength(1) + }) +}) + +describe('STA-3077: superseding respects the durable pane binding', () => { + // Spawn-shaped setup: the creating branches are intentionally allowed here. + function bindPaneTo(store: Awaited>, ptyId: string) { + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId, + incarnationId: `inc-${ptyId}` + }) + } + + it('scrubs the predecessor binding when the arriving lease is the bound one', async () => { + const store = await createStore() + bindPaneTo(store, 'relay-pty-a') + store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1)) + bindPaneTo(store, 'relay-pty-b') + store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2)) + + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-b']) + const layout = store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB] + expect(layout?.ptyIdsByLeafId?.[LEAF]).toBe('relay-pty-b') + }) + + // Expiring the bound predecessor here would detach a live pane, so both stay + // live and reattach arbitrates with the binding in hand. + it('defers instead of expiring the lease the pane is bound to', async () => { + const store = await createStore() + bindPaneTo(store, 'relay-pty-a') + store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1)) + store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2)) + + expect( + liveLeasesForPane(store) + .map((lease) => lease.ptyId) + .sort() + ).toEqual(['relay-pty-a', 'relay-pty-b']) + expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe( + 'relay-pty-a' + ) + + // Arbitration keeps the bound lease, not the newer one. + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1) + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-a']) + }) +}) + +describe('STA-3077: existing duplicate leases are healed, not revived', () => { + // Installs that predate pane-keyed supersession already carry the duplicates + // this bug accumulated. Preventing new ones does not help them. + it('retires every stale duplicate for a pane and keeps the newest', async () => { + const store = await createStore({ + sshRemotePtyLeases: Array.from({ length: 20 }, (_, index) => ({ + ...leaseFor(`relay-pty-${index}`, index + 1), + createdAt: index + 1 + })) + }) + + const retired = await store.supersedeDuplicatePaneLeases(TARGET) + + expect(retired).toBe(19) + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-19']) + }) + + // Recency alone would retire the lease the pane is actually bound to whenever a + // newer unbound lease exists, detaching a live pane instead of healing it. + it('keeps the durably bound lease even when an unbound one is newer', async () => { + const store = await createStore() + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-bound', + incarnationId: 'inc-bound' + }) + store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-bound', 1), createdAt: 1 }) + // Arrives later but no pane is bound to it. + store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-newer', 99), createdAt: 99 }) + + await store.supersedeDuplicatePaneLeases(TARGET) + + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-bound']) + }) + + // A retirement that is not durable must not be believed: it would read as + // retired in memory and attached on disk for the rest of the session. + it('rolls the retirement back when the durable write fails', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + // The durable write is the async twin: retirement runs on reconnect, and the sync flush + // fsyncs a multi-MB file from the main thread. Reaching for a private name is not ideal, + // but the property under test is the rollback, and it must follow whichever writer the + // retirement actually awaits. + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockRejectedValue(new Error('disk full')) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(0) + expect(liveLeasesForPane(store)).toHaveLength(2) + }) + + it('does not chase store mutations after the retirement generation is durable', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + const internal = store as unknown as { enqueueWrite: () => Promise } + const enqueueWrite = internal.enqueueWrite.bind(store) + let injectedMutation = false + const enqueueSpy = vi.spyOn(internal, 'enqueueWrite').mockImplementation(async () => { + await enqueueWrite() + if (!injectedMutation) { + injectedMutation = true + store.updateUI({ sidebarWidth: 777 }) + } + }) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1) + + expect(enqueueSpy).toHaveBeenCalledOnce() + enqueueSpy.mockRestore() + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) + + it('retries when a mutation invalidates the retirement write before rename', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + const internal = store as unknown as { writeToDiskAsync: () => Promise } + const writeToDiskAsync = internal.writeToDiskAsync.bind(store) + let injectedMutation = false + const writeSpy = vi.spyOn(internal, 'writeToDiskAsync').mockImplementation(async () => { + const write = writeToDiskAsync() + if (!injectedMutation) { + injectedMutation = true + store.updateUI({ sidebarWidth: 778 }) + } + await write + }) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1) + + expect(writeSpy).toHaveBeenCalledTimes(2) + }) + + it('does not undo a concurrent confirmation of the provisional expired state', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'expired') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('expired') + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) + + it('rolls back after a rejected concurrent attach request', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'attached') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('attached') + }) + + it('does not revive a provisional loser after a concurrent same-pane upsert', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-c', 3), createdAt: 3 }) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-c']) + }) + + it('does not restore attached state over a concurrent detach', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLeasesForShutdown(TARGET, 'detached') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('expired') + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-b')?.state + ).toBe('detached') + }) + + it('does not roll back concurrent session or lease changes after a failed write', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'terminated') + const concurrentTab = 'tab-concurrent' + const concurrentLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: concurrentTab, + leafId: concurrentLeaf, + ptyId: 'relay-pty-concurrent', + incarnationId: 'inc-concurrent' + }) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('terminated') + expect( + store.getWorkspaceSession().terminalLayoutsByTabId?.[concurrentTab]?.ptyIdsByLeafId?.[ + concurrentLeaf + ] + ).toBe('relay-pty-concurrent') + }) + + it('does not restore a binding for a lease removed during a failed write', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a' + }) + delete store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF] + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.removeSshRemotePtyLease(TARGET, 'relay-pty-a') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect(store.getSshRemotePtyLeases(TARGET).some((lease) => lease.ptyId === 'relay-pty-a')).toBe( + false + ) + expect( + store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)?.ptyId + ).toBeNull() + }) + + it('restores a tab binding into the moved leaf after a concurrent session replacement', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a' + }) + delete store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF] + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + expect( + store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)?.ptyId + ).toBeNull() + const movedTabId = 'tab-moved' + const replacement = structuredClone(store.getWorkspaceSession()) + const movedTab = replacement.tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB) + if (!movedTab || !replacement.terminalLayoutsByTabId?.[TAB]) { + throw new Error('expected source tab and layout') + } + movedTab.id = movedTabId + replacement.terminalLayoutsByTabId[movedTabId] = replacement.terminalLayoutsByTabId[TAB] + delete replacement.terminalLayoutsByTabId[TAB] + store.setWorkspaceSession(replacement) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === movedTabId) + ?.ptyId + ).toBe('relay-pty-a') + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) + + it('does not restore a retired binding after the leaf is deleted', async () => { + const otherTab = 'tab-other' + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2, tabId: otherTab } + ] + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a' + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: otherTab, + leafId: LEAF, + ptyId: 'relay-pty-b', + incarnationId: 'inc-b' + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe( + undefined + ) + const replacement = structuredClone(store.getWorkspaceSession()) + replacement.tabsByWorktree![WORKTREE] = [] + replacement.terminalLayoutsByTabId = {} + store.setWorkspaceSession(replacement) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect(store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]).toEqual([]) + expect(store.getWorkspaceSession().terminalLayoutsByTabId).toEqual({}) + }) + + it('leaves distinct panes alone', async () => { + const otherLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2, leafId: otherLeaf } + ] + }) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(0) + expect(store.getSshRemotePtyLeases(TARGET).filter((l) => l.state === 'attached')).toHaveLength( + 2 + ) + }) + + // A lease is rebuilt field by field on load, so a field added to the type but not to the + // normalizer is dropped on every boot — it would look right in memory and be gone on the next + // start, which is how a fence ships silently permitting everything. A synthesized identity is + // refused outright rather than carried: it is not stable across reconnects, so one persisted by + // another build would later read as a different shell and strand a live pane. + it('keeps a host-attested incarnation across a reload and refuses a synthesized one', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1, incarnationId: 'inc-host-a' }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2, incarnationId: 'legacy:23:0:relay-pty-b' } + ] + }) + + const byPtyId = new Map(store.getSshRemotePtyLeases(TARGET).map((l) => [l.ptyId, l])) + expect(byPtyId.get('relay-pty-a')?.incarnationId).toBe('inc-host-a') + expect(byPtyId.get('relay-pty-b')?.incarnationId).toBeUndefined() + }) +}) + +describe('SSH reconnect lease retirement batching', () => { + it('keeps concurrent terminal state when the asynchronous write fails', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['relay-pty-a']) + const concurrentTab = 'tab-concurrent' + const concurrentLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: concurrentTab, + leafId: concurrentLeaf, + ptyId: 'relay-pty-concurrent', + incarnationId: 'inc-concurrent' + }) + rejectWrite(new Error('disk full')) + + await expect(retirement).rejects.toThrow('disk full') + expect( + store.getWorkspaceSession().terminalLayoutsByTabId?.[concurrentTab]?.ptyIdsByLeafId?.[ + concurrentLeaf + ] + ).toBe('relay-pty-concurrent') + }) +}) + +describe('STA-3077: reattach binds panes, it never creates them', () => { + // RC3: persistPtyBinding has four creating branches (mint tab, mint root leaf, + // split root and graft leaf, mint layout). They are load-bearing for spawn and + // wrong for reattach, where the pane either exists or is gone for good. + it('does not mint a tab for a pane that no longer exists', async () => { + const store = await createStore() + + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a', + mayCreate: false + }) + + const session = store.getWorkspaceSession() + expect(session.tabsByWorktree?.[WORKTREE] ?? []).toHaveLength(0) + expect(session.terminalLayoutsByTabId?.[TAB]).toBeUndefined() + }) + + // Unknown is not dead: failing to resolve a pane must not be reported as + // success, and must not terminate anything. + it('reports an unresolved reattach rather than silently succeeding', async () => { + const store = await createStore() + + const bound = store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a', + mayCreate: false + }) + + expect(bound).toBe(false) + }) +}) + +describe('STA-3077: exact-binding compare-and-swap', () => { + // Already correct at HEAD. Pinned so a fix cannot regress it: a stale renderer + // replay must not overwrite a binding the host has since re-admitted. + it('refuses a write whose expected binding no longer matches', async () => { + const store = await createStore() + + const stale = store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-b', + incarnationId: 'inc-b', + expectedBinding: { ptyId: 'relay-pty-a', incarnationId: 'inc-a' } + }) + + expect(stale).toBe(false) + }) +}) + +describe('STA-3077 step F: live-layout tab resolution is reattach-only', () => { + // The lease's tabId is the frozen side on reattach, so the live layout outranks it there. On a + // SPAWN the opposite holds: the caller's tabId is fresh truth and the persisted layout is the + // stale side, because the renderer publishes membership on a debounce. Breaking a pane out into + // a new tab and spawning into it inside that window would otherwise resolve back to the tab the + // pane just left, writing the durable binding and the fence under one tab while the lease and + // the runtime registration use the other — the split-coordinate defect step F exists to remove. + it('resolves the live tab only on the reattach bind, never on a spawn', async () => { + const { readFileSync } = await import('node:fs') + const relay = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + const ipc = readFileSync('src/main/ipc/pty.ts', 'utf-8') + + // The reattach holds a tabId frozen in a lease, so it must resolve the live one first. + expect(relay).toContain('resolvePaneShellTabId(') + // Resolving BEFORE the durable write is what keeps a thrown write from losing the answer and + // registering the pane in the graph under the tab it left. + expect(relay.indexOf('resolvePaneShellTabId(')).toBeLessThan(relay.indexOf('bindPaneShell({')) + // Spawn callers hold the fresh tabId; none of them may consult the debounced layout. + expect(ipc).toContain('bindPaneShell({') + expect(ipc.split('bindPaneShell({').slice(1).join('')).not.toContain('resolvePaneShellTabId') + }) +}) + +describe('STA-3077: the reattach path actually refuses to create', () => { + // This is the oracle the store-level tests could not provide: `mayCreate` + // existed and was correct, but no production caller passed it, so reattach + // still grafted panes. Pin the wiring, not just the capability. + // The reattach bind now goes through the one `bindPaneShell` producer, and it + // still refuses to create. + it('passes mayCreate:false from the SSH reattach binding write', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + const bindCall = source.slice( + source.indexOf('restoreReattachedPtyRuntime'), + source.indexOf('private async attachPtyWithRetry') + ) + expect(bindCall).toContain('bindPaneShell') + expect(bindCall).toContain('mayCreate: false') + }) + + // Strengthened, not relaxed: counting guards against calls went vacuous once + // the direct store calls disappeared, so pin both halves instead. + it('has no production persistPtyBinding caller that can create during reattach', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + // Every bind in the relay session is a reattach; none may grow topology. + expect(source.split('persistPtyBinding(').length - 1).toBe(0) + const binds = source.split('bindPaneShell(').length - 1 + expect(binds).toBeGreaterThan(0) + // Each `bindPaneShell(` call site must carry its own `mayCreate: false`. + const guardedBinds = source + .split('bindPaneShell(') + .slice(1) + .filter((tail) => tail.slice(0, tail.indexOf('})')).includes('mayCreate: false')).length + expect(guardedBinds).toBe(binds) + }) +}) diff --git a/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts b/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts index 9d08ac7da0f..9de9e9f1100 100644 --- a/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts +++ b/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts @@ -69,6 +69,7 @@ vi.mock('../providers/ssh-git-provider', () => ({ SshGitProvider: class MockSshGitProvider {} })) vi.mock('../ipc/pty', () => ({ + resolvePaneShellTabId: vi.fn(() => undefined), registerSshPtyProvider: vi.fn(), unregisterSshPtyProvider: vi.fn(), getSshPtyProvider: vi.fn(), @@ -186,24 +187,24 @@ describe('SshRelaySession abandoned remote PTYs', () => { expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID) }) - it('retires the lease without a kill when the relay proves the PTY is gone', async () => { - // pty.attach verifies process liveness before answering not-found, so this is the one branch - // with positive proof of death — and a dead process needs no shutdown request. + // INVERTED for STA-3077 step E-0. This case previously asserted that a bare not-found retires the + // lease and reports `pty:exit { code: -1 }`, on the premise that attach proves liveness before + // answering not-found. It does not: the relay answers not-found for a pane-identity mismatch and + // for any id it merely cannot hand back, so the premise licensed a fabricated death certificate. + // The clause is kept — inverted — so the new intent stays covered rather than silently dropped. + it('leaves the shell running when the relay only reports the PTY as not found', async () => { const { deps, shutdown } = await establishWithFailingReattach( new Error('PTY "pty-live" not found') ) expect(shutdown).not.toHaveBeenCalled() - expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( 'target-1', 'pty-live', 'expired' ) - expect(clearProviderPtyState).toHaveBeenCalledWith(APP_PTY_ID) - expect(deps.mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { - id: APP_PTY_ID, - code: -1 - }) + expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID) + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) }) it('keeps a recovered session attached when reattach succeeds after an earlier drop', async () => { diff --git a/src/main/ssh/ssh-relay-reattach-exit-proof.test.ts b/src/main/ssh/ssh-relay-reattach-exit-proof.test.ts new file mode 100644 index 00000000000..dc4bae2e8d2 --- /dev/null +++ b/src/main/ssh/ssh-relay-reattach-exit-proof.test.ts @@ -0,0 +1,358 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { formatPtyExitedError } from '../../shared/ssh-pty-failure-tokens' +import { SshRelaySession } from './ssh-relay-session' +import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' + +const { acceptOutputExitMock, muxRequestMock } = vi.hoisted(() => ({ + acceptOutputExitMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn() +})) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: vi.fn( + async (_mux: unknown, options: { clientInstanceId: string }) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + }) + ) +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: vi.fn().mockResolvedValue(undefined), + acceptSshPtyOutputExit: acceptOutputExitMock, + allocateSshPtyProviderGeneration: vi.fn(() => 17), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + applySshPtySourceRecoveryCancellationProof: vi.fn(() => true), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}) +})) +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) +vi.mock('./ssh-remote-orca-cli', () => ({ + runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }) +})) +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = muxRequestMock + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = vi.fn() + isDisposed = vi.fn().mockReturnValue(false) + } +})) +vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({ + installRemoteManagedAgentHooks: vi.fn().mockResolvedValue([]) +})) +vi.mock('../providers/ssh-pty-provider', () => ({ + SshPtyProvider: class MockSshPtyProvider { + onData = vi.fn().mockReturnValue(() => {}) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockReturnValue(() => {}) + attach = vi.fn().mockResolvedValue(undefined) + attachForReconnect = vi.fn().mockResolvedValue({}) + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../ipc/pty', () => ({ + resolvePaneShellTabId: vi.fn(() => undefined), + registerSshPtyProvider: vi.fn(), + unregisterSshPtyProvider: vi.fn(), + getSshPtyProvider: vi.fn(), + getPtyIdsForConnection: vi.fn().mockReturnValue([]), + clearPtyOwnershipForConnection: vi.fn(), + clearProviderPtyState: vi.fn(), + deletePtyOwnership: vi.fn(), + setPtyOwnership: vi.fn(), + restorePtyIncarnation: vi.fn(), + isCurrentPtyExit: vi.fn(() => true), + answerStartupTerminalColorQueriesForPty: vi.fn((_id: string, data: string) => data) +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { + registerSshPtyProvider, + getSshPtyProvider, + getPtyIdsForConnection, + clearProviderPtyState, + deletePtyOwnership +} = await import('../ipc/pty') + +const TARGET_ID = 'target-1' +const RELAY_PTY_ID = 'pty-live' +const APP_PTY_ID = 'ssh:target-1@@pty-live' +const LEAF_ID = '11111111-1111-4111-8111-111111111111' +const PROVIDER_GENERATION = 17 + +// The relay's own not-found wording; `isSshPtyNotFoundError` matches it and nothing narrows it +// further, so it arrives carrying no evidence about whether the shell is alive. +const UNPROVEN_NOT_FOUND = `PTY "${RELAY_PTY_ID}" not found` + +function detachedLease() { + return { + targetId: TARGET_ID, + ptyId: RELAY_PTY_ID, + state: 'detached' as const, + worktreeId: 'worktree-1', + tabId: 'tab-1', + leafId: LEAF_ID, + incarnationId: 'inc-proven-exit' + } +} + +/** + * STA-3077 / design E-0: a reattach that ends in a bare not-found tells us the relay cannot hand + * this id back — never that the process died. Synthesizing `pty:exit { code: -1 }` from it is a + * fabricated death certificate: the renderer treats it as a real exit and the surviving remote + * shell (and any agent inside it) is written off. Only an exit the relay actually observed may + * report an exit; everything else routes into the non-destructive detached branch. + */ +describe('SshRelaySession reattach failures never fabricate an exit', () => { + let warn: ReturnType + + beforeEach(() => { + vi.clearAllMocks() + muxRequestMock.mockReset() + muxRequestMock.mockResolvedValue([]) + acceptOutputExitMock.mockResolvedValue(undefined) + mockDeploySuccess() + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'info').mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + /** Drives the real connect-path reattach loop, so the failure reaches the handler as production does. */ + async function reattachFailsWith(message: string): Promise<{ + deps: ReturnType + attachForReconnect: ReturnType + }> { + const deps = createMockDeps() + const attachForReconnect = vi.fn().mockRejectedValue(new Error(message)) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect, + shutdown: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([detachedLease()] as ReturnType< + typeof deps.mockStore.getSshRemotePtyLeases + >) + const session = new SshRelaySession( + TARGET_ID, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + + await session.establish(deps.mockConn) + // Precondition, not a claim: if the reattach never ran, every "did not destroy" clause below + // would pass for the wrong reason. + expect(attachForReconnect).toHaveBeenCalled() + + return { deps, attachForReconnect } + } + + function warnings(): string[] { + return warn.mock.calls.map((call) => String(call[0])) + } + + it('reaches the failure handler through the real reattach path', async () => { + // Producer pin: without this the remaining clauses could pass on a path production never takes. + const { attachForReconnect } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(attachForReconnect.mock.calls[0]?.[0]).toBe(RELAY_PTY_ID) + }) + + it('sends no synthetic pty:exit to the pane on an unproven not-found', async () => { + const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + }) + + it('keeps pty ownership on an unproven not-found', async () => { + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deletePtyOwnership).not.toHaveBeenCalledWith(APP_PTY_ID) + }) + + it('keeps provider pty state on an unproven not-found', async () => { + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID) + }) + + it('leaves the lease terminable instead of expiring it on an unproven not-found', async () => { + const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + TARGET_ID, + RELAY_PTY_ID, + 'expired' + ) + }) + + // The counterpart. An exit the relay watched settles it, so the record is retired rather than + // re-attached on every future reconnect for the life of the install. `terminated`, not `expired`: + // expiry is the state the recovery grant reads, and retiring a record must not also authorise a + // replacement. + it('retires the lease on an exit the relay proved', async () => { + const { deps } = await reattachFailsWith( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-proven-exit') + ) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledWith(TARGET_ID, [ + RELAY_PTY_ID + ]) + }) + + it('persists a reconnect batch of proven exits in one async write', async () => { + const deps = createMockDeps() + const secondPtyId = 'pty-second' + const secondIncarnationId = 'inc-proven-second-exit' + const attachForReconnect = vi.fn().mockImplementation((ptyId: string) => { + const incarnationId = ptyId === RELAY_PTY_ID ? 'inc-proven-exit' : secondIncarnationId + return Promise.reject(new Error(formatPtyExitedError(ptyId, 0, incarnationId))) + }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect, + shutdown: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([ + detachedLease(), + { + ...detachedLease(), + ptyId: secondPtyId, + leafId: '22222222-2222-4222-8222-222222222222', + incarnationId: secondIncarnationId + } + ] as ReturnType) + const session = new SshRelaySession( + TARGET_ID, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + + await session.establish(deps.mockConn) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledOnce() + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledWith(TARGET_ID, [ + RELAY_PTY_ID, + secondPtyId + ]) + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalled() + }) + + it('does not retire the lease for proof naming another PTY', async () => { + const { deps } = await reattachFailsWith( + formatPtyExitedError('pty-some-other-shell', 0, 'inc-proven-exit') + ) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled() + }) + + it('does not retire the lease for proof naming another incarnation', async () => { + const { deps } = await reattachFailsWith( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-some-other-shell') + ) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled() + }) + + it('does not retire the lease when the failure proves nothing', async () => { + const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled() + }) + + it('routes an unproven not-found into the non-destructive detached branch', async () => { + // That branch announces itself with the "leaving detached" warning; the destructive one says + // "dropping stale". The log is how the two are told apart from outside the session. + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(warnings().some((line) => line.includes(RELAY_PTY_ID) && /detached/i.test(line))).toBe( + true + ) + }) + + it('does not take the drop-stale branch on an unproven not-found', async () => { + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(warnings().some((line) => /Dropping stale PTY/i.test(line))).toBe(false) + }) + + it('still reports an exit the relay actually observed', async () => { + // Contrast: the only evidence that authorizes a death claim is an exit the relay saw on a live + // stream. Removing the fabricated one must not mute this one. + const deps = createMockDeps() + vi.mocked(getSshPtyProvider).mockReturnValue({ + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue( + [] as ReturnType + ) + const session = new SshRelaySession( + TARGET_ID, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + await session.establish(deps.mockConn) + const provider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { + onExit: ReturnType + } + const onExit = provider.onExit.mock.calls[0]?.[0] as (payload: unknown) => void + + onExit({ + id: APP_PTY_ID, + code: 3, + incarnationId: 'incarnation-1', + providerGeneration: PROVIDER_GENERATION, + ptyIncarnation: 'incarnation-1' + }) + + await vi.waitFor(() => + expect(acceptOutputExitMock).toHaveBeenCalledWith({ + id: APP_PTY_ID, + code: 3, + providerGeneration: PROVIDER_GENERATION, + ptyIncarnation: 'incarnation-1' + }) + ) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( + TARGET_ID, + RELAY_PTY_ID, + 'terminated' + ) + expect(deletePtyOwnership).toHaveBeenCalledWith(APP_PTY_ID) + }) +}) diff --git a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts index f54afef3f89..17952f4ade0 100644 --- a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts +++ b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts @@ -156,6 +156,7 @@ function createSession(targetId: string): InstanceType { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), diff --git a/src/main/ssh/ssh-relay-session-data-delivery.test.ts b/src/main/ssh/ssh-relay-session-data-delivery.test.ts index 2eca6802f28..2a6edf1840e 100644 --- a/src/main/ssh/ssh-relay-session-data-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-data-delivery.test.ts @@ -403,7 +403,6 @@ describe('SshRelaySession data delivery', () => { expect(retryCalls[0]).toHaveProperty('resume') expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, Object.freeze({ status: 'checkpointUnavailable' }) ) second.dispose() @@ -762,7 +761,6 @@ describe('SshRelaySession data delivery', () => { expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, expect.objectContaining({ status: 'checkpoint', deliveryToken: 'old-token', diff --git a/src/main/ssh/ssh-relay-session-model-migration.test.ts b/src/main/ssh/ssh-relay-session-model-migration.test.ts index ae5db55e0cc..ab1512d82dc 100644 --- a/src/main/ssh/ssh-relay-session-model-migration.test.ts +++ b/src/main/ssh/ssh-relay-session-model-migration.test.ts @@ -199,7 +199,6 @@ describe('SshRelaySession model migration', () => { expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, Object.freeze({ status: 'checkpoint', clientGeneration: 1, @@ -259,7 +258,6 @@ describe('SshRelaySession model migration', () => { expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, Object.freeze({ status: 'checkpointUnavailable' }) ) }) diff --git a/src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts b/src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts new file mode 100644 index 00000000000..5c52aac8ea9 --- /dev/null +++ b/src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts @@ -0,0 +1,334 @@ +/** + * The superseded-PTY fence must be LIVE on the relay reattach path — the path it + * was built for. Production wiring only: the real `pty:spawn`/`pty:write` + * handlers from src/main/ipc/pty.ts and a real `SshRelaySession` whose reconnect + * drives `restoreReattachedPtyRuntime`; only the wire below the provider is fake. + * + * Today reattach binds the pane through `runtime.registerPty` while spawn binds + * through `rememberPaneKeyForPty`, so the fence maps never learn the successor + * and a keystroke queued for the superseded PTY still reaches the host. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { makePaneKey } from '../../shared/stable-pane-id' + +const handlers = new Map unknown>() +const listeners = new Map unknown>() + +vi.mock('electron', () => ({ + BrowserWindow: undefined, + app: { + isPackaged: true, + getPath: () => '/tmp/orca-reattach-fence-test', + getVersion: () => '0.0.0-test' + }, + powerMonitor: { on: vi.fn() }, + nativeTheme: { shouldUseDarkColors: true }, + ipcMain: { + handle: (channel: string, handler: (event: unknown, args: unknown) => unknown) => { + handlers.set(channel, handler) + }, + on: (channel: string, listener: (event: unknown, args: unknown) => unknown) => { + listeners.set(channel, listener) + }, + removeHandler: (channel: string) => handlers.delete(channel), + removeAllListeners: (channel: string) => listeners.delete(channel) + } +})) + +const { providerInstances, spawnedPtyIds, attachIncarnationId } = vi.hoisted(() => ({ + providerInstances: [] as unknown[], + spawnedPtyIds: [] as string[], + attachIncarnationId: 'incarnation-reattached' +})) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) +vi.mock('./ssh-remote-orca-cli', () => ({ + runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }) +})) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: vi.fn( + async (_mux: unknown, options: { clientInstanceId: string }) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + }) + ) +})) +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = vi.fn().mockResolvedValue([]) + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = vi.fn() + isDisposed = vi.fn().mockReturnValue(false) + } +})) +vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({ + installRemoteManagedAgentHooks: vi.fn() +})) +// The relay session constructs and registers this provider itself, so the real +// pty.ts registry hands the same instance to pty:spawn and pty:write. +vi.mock('../providers/ssh-pty-provider', () => ({ + isSshPtyNotFoundError: (error: unknown) => String(error).includes('not found'), + isSshPtyIdentityMismatchError: (error: unknown) => String(error).includes('identity mismatch'), + SshPtyProvider: class MockSshPtyProvider { + spawn = vi.fn(async () => ({ id: spawnedPtyIds.shift() ?? 'ssh:target-1@@pty-unexpected' })) + write = vi.fn() + resize = vi.fn() + shutdown = vi.fn() + sendSignal = vi.fn() + getCwd = vi.fn() + getInitialCwd = vi.fn() + clearBuffer = vi.fn() + acknowledgeDataEvent = vi.fn() + hasChildProcesses = vi.fn() + getForegroundProcess = vi.fn() + serialize = vi.fn() + revive = vi.fn() + listProcesses = vi.fn(async () => []) + getDefaultShell = vi.fn() + getProfiles = vi.fn() + onData = vi.fn().mockReturnValue(() => {}) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockReturnValue(() => {}) + attach = vi.fn().mockResolvedValue(undefined) + attachForReconnect = vi.fn().mockResolvedValue({ incarnationId: attachIncarnationId }) + dispose = vi.fn() + + constructor() { + providerInstances.push(this) + } + } +})) +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { SshRelaySession } = await import('./ssh-relay-session') +const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') +const { + registerPtyHandlers, + unregisterSshPtyProvider, + getPtyIdForPaneKey, + getPtyIdsForConnection, + deletePtyOwnership +} = await import('../ipc/pty') + +const TARGET = 'target-1' +const TAB_ID = 'tab-1' + +type ProviderMock = { write: ReturnType } + +function makeMainWindow() { + const webContents = { + id: 1, + send: vi.fn(), + on: vi.fn(), + once: vi.fn(), + removeListener: vi.fn(), + isDestroyed: () => false + } + return { isDestroyed: () => false, isVisible: () => true, isMinimized: () => false, webContents } +} + +/** A live layout in which `leafId` sits in `tabId` — the tab the pane is in RIGHT NOW. + * The tab must also exist in `tabsByWorktree`: resolution ignores a layout whose tab is gone, so + * a fixture without one would resolve nothing and the clause would pass for the wrong reason. */ +function sessionWithLeafInTab(tabId: string, leafId: string) { + return { + tabsByWorktree: { 'worktree-1': [{ id: tabId, worktreeId: 'worktree-1' }] }, + terminalLayoutsByTabId: { + [tabId]: { root: { type: 'leaf' as const, leafId }, ptyIdsByLeafId: {} } + } + } +} + +function makeStore(leases: unknown[], session?: ReturnType) { + return { + ...(session ? { getWorkspaceSession: vi.fn(() => session) } : {}), + getRepos: vi.fn().mockReturnValue([]), + getSshPtyConsumerRecovery: vi.fn().mockReturnValue(null), + upsertSshPtyConsumerRecovery: vi.fn(), + removeSshPtyConsumerRecovery: vi.fn(), + getSshRemotePtyLeases: vi.fn(() => leases), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), + markSshRemotePtyLease: vi.fn(), + markSshRemotePtyLeases: vi.fn(), + markSshRemotePtyLeasesAsync: vi.fn(), + markSshRemotePtyLeasesForShutdown: vi.fn(), + markSshRemotePtyLeasesAttachedAsync: vi.fn(), + upsertSshRemotePtyLease: vi.fn(), + persistPtyBinding: vi.fn().mockReturnValue(true) + } +} + +/** + * One pane: spawned over SSH, then rebound by a reconnect whose durable lease + * names a different remote PTY — the STA-3077 shape. + */ +async function spawnThenReattachPane(leafId: string, currentTabId?: string) { + const mainWindow = makeMainWindow() + const predecessorPtyId = `ssh:${TARGET}@@pty-old-${leafId.slice(0, 4)}` + const reattachedRelayPtyId = `pty-new-${leafId.slice(0, 4)}` + const leases: unknown[] = [] + const store = makeStore( + leases, + currentTabId ? sessionWithLeafInTab(currentTabId, leafId) : undefined + ) + const runtime = { onPtySpawned: vi.fn(), registerPty: vi.fn() } + const session = new SshRelaySession( + TARGET, + vi.fn().mockReturnValue(mainWindow) as never, + store as never, + { removeAllForwards: vi.fn() } as never, + runtime as never + ) + + registerPtyHandlers( + mainWindow as never, + undefined, + undefined, + undefined, + undefined, + store as never + ) + await session.establish({} as never) + + spawnedPtyIds.push(predecessorPtyId) + await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + env: { ORCA_PANE_KEY: makePaneKey(TAB_ID, leafId) }, + connectionId: TARGET, + worktreeId: 'worktree-1', + tabId: TAB_ID, + leafId + }) + + // The relay restarted; the pane's durable lease now names a different shell. + leases.push({ + targetId: TARGET, + ptyId: reattachedRelayPtyId, + state: 'detached' as const, + worktreeId: 'worktree-1', + tabId: TAB_ID, + leafId + }) + await session.reconnect({} as never) + // Reconnect builds a fresh provider; writes route through the live one. + const provider = providerInstances.at(-1) as ProviderMock + + const write = (id: string): void => { + listeners.get('pty:write')!({ sender: mainWindow.webContents }, { id, data: 'x' }) + } + return { + runtime, + paneKey: makePaneKey(currentTabId ?? TAB_ID, leafId), + predecessorPtyId, + reattachedPtyId: `ssh:${TARGET}@@${reattachedRelayPtyId}`, + provider, + write, + dispose: () => { + deletePtyOwnership(predecessorPtyId) + deletePtyOwnership(`ssh:${TARGET}@@${reattachedRelayPtyId}`) + unregisterSshPtyProvider(TARGET) + } + } +} + +describe('a relay reattach binds the pane through the same producer as spawn', () => { + let cleanup: (() => void) | undefined + + beforeEach(() => { + handlers.clear() + listeners.clear() + providerInstances.length = 0 + spawnedPtyIds.length = 0 + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, + platform: 'linux-x64' + } as never) + }) + + afterEach(() => { + cleanup?.() + cleanup = undefined + }) + + it('refuses a queued write to the PTY the reattach superseded', async () => { + const pane = await spawnThenReattachPane('11111111-1111-4111-8111-111111111111') + cleanup = pane.dispose + pane.provider.write.mockClear() + // Still an owned, routable PTY — so only the fence can refuse the write. + expect(getPtyIdsForConnection(TARGET)).toContain(pane.predecessorPtyId) + + // A keystroke the renderer queued before the reconnect, still addressed to + // the shell the pane no longer owns. + pane.write(pane.predecessorPtyId) + await Promise.resolve() + + expect(pane.provider.write).not.toHaveBeenCalled() + }) + + it('resolves the pane to the reattached PTY', async () => { + const pane = await spawnThenReattachPane('22222222-2222-4222-8222-222222222222') + cleanup = pane.dispose + + expect(getPtyIdForPaneKey(pane.paneKey)).toBe(pane.reattachedPtyId) + }) + + // The lease's tabId is frozen at write time, but `detachTerminalPaneToTab` moves a live pane + // into a new tab without killing its PTY. A producer that forwarded `lease.tabId` would key the + // fence to the tab the pane LEFT — and every other clause here uses one tab on both sides, so + // nothing else in this file would notice. + it('keys the pane to the tab the leaf is in now, not the one its lease was written in', async () => { + const leafId = '44444444-4444-4444-8444-444444444444' + const pane = await spawnThenReattachPane(leafId, 'tab-moved-to') + cleanup = pane.dispose + + expect(pane.paneKey).toBe(makePaneKey('tab-moved-to', leafId)) + expect(getPtyIdForPaneKey(pane.paneKey)).toBe(pane.reattachedPtyId) + expect(getPtyIdForPaneKey(makePaneKey(TAB_ID, leafId))).not.toBe(pane.reattachedPtyId) + // The runtime graph must land on the SAME tab as the record and the fence. Registering under + // the lease's frozen tabId would split the pane across two tabs and ensure a mobile surface + // for the one it left. + expect(pane.runtime.registerPty).toHaveBeenCalledWith( + pane.reattachedPtyId, + 'worktree-1', + TARGET, + expect.objectContaining({ tabId: 'tab-moved-to', leafId }) + ) + }) + + // Guards the clause above from a fence that simply refuses everything. + it('still delivers a write to the reattached PTY', async () => { + const pane = await spawnThenReattachPane('33333333-3333-4333-8333-333333333333') + cleanup = pane.dispose + pane.provider.write.mockClear() + + pane.write(pane.reattachedPtyId) + await Promise.resolve() + + expect(pane.provider.write).toHaveBeenCalledWith(pane.reattachedPtyId, 'x') + }) +}) diff --git a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts index 3eebb3574b3..b958c26db1b 100644 --- a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts @@ -101,6 +101,9 @@ vi.mock('../providers/ssh-git-provider', () => ({ SshGitProvider: class MockSshGitProvider {} })) vi.mock('../ipc/pty', () => ({ + resolvePaneShellTabId: vi.fn(() => undefined), + // Step F: the single pane->shell bind producer the relay now calls. + bindPaneShell: vi.fn(() => true), registerSshPtyProvider: vi.fn(), unregisterSshPtyProvider: vi.fn(), getSshPtyProvider: vi.fn(), @@ -124,6 +127,7 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ })) const { + bindPaneShell, registerSshPtyProvider, getSshPtyProvider, getPtyIdsForConnection, @@ -435,18 +439,43 @@ describe('SshRelaySession reconnect incarnation ordering', () => { incarnationId }) expect(runtime.onPtySpawned).not.toHaveBeenCalled() - expect(mockStore.persistPtyBinding).toHaveBeenCalledWith({ + // Step F moved the observation point to the one bind producer; the intent is + // unchanged — exact incarnation proof, and mayCreate:false so reattach binds + // an existing pane and never grafts one back. + expect(bindPaneShell).toHaveBeenCalledWith({ + store: expect.anything(), worktreeId: 'worktree-1', tabId: 'tab-1', leafId: INCARNATION_LEAF_ID, ptyId: APP_PTY_ID, - incarnationId + incarnationId, + mayCreate: false }) - expect(vi.mocked(mockStore.persistPtyBinding).mock.invocationCallOrder[0]).toBeLessThan( + expect(vi.mocked(bindPaneShell).mock.invocationCallOrder[0]).toBeLessThan( vi.mocked(mockStore.markSshRemotePtyLeasesAttachedAsync).mock.invocationCallOrder[0]! ) }) + it('sends pane identity beside incarnation for relays that only understand the legacy fence', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const attachForReconnect = vi.fn().mockResolvedValue({ incarnationId: 'incarnation-lease' }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([ + { ...detachedLease(), incarnationId: 'incarnation-lease' } + ] as ReturnType) + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + + await session.establish(mockConn) + + expect(attachForReconnect).toHaveBeenCalledWith('pty-live', undefined, 'incarnation-lease', { + paneKey: `tab-1:${INCARNATION_LEAF_ID}`, + tabId: 'tab-1' + }) + }) + it('does not restore a PTY whose matching exit shares the attach reply batch', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const incarnationId = 'incarnation-exited-during-attach' @@ -489,7 +518,8 @@ describe('SshRelaySession reconnect incarnation ordering', () => { expect(runtime.registerPty).not.toHaveBeenCalled() expect(restorePtyIncarnation).toHaveBeenCalledWith(APP_PTY_ID, incarnationId) expect(setPtyOwnership).not.toHaveBeenCalled() - expect(mockStore.persistPtyBinding).not.toHaveBeenCalled() + // Step F: observed at the one bind producer, else this negative is vacuous. + expect(bindPaneShell).not.toHaveBeenCalled() expect(sourceActivationLease.rollback).toHaveBeenCalledOnce() expect(sourceActivationLease.commit).not.toHaveBeenCalled() expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( @@ -548,8 +578,14 @@ describe('SshRelaySession reconnect incarnation ordering', () => { incarnationId: currentIncarnationId }) expect(setPtyOwnership).toHaveBeenCalledWith(APP_PTY_ID, 'target-1') - expect(mockStore.persistPtyBinding).toHaveBeenCalledWith( - expect.objectContaining({ ptyId: APP_PTY_ID, incarnationId: currentIncarnationId }) + // Step F moved the observation point to the one bind producer; the asserted + // intent — exact incarnation proof, mayCreate:false — is unchanged. + expect(bindPaneShell).toHaveBeenCalledWith( + expect.objectContaining({ + ptyId: APP_PTY_ID, + incarnationId: currentIncarnationId, + mayCreate: false + }) ) expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:replay', { id: APP_PTY_ID, @@ -567,7 +603,9 @@ describe('SshRelaySession reconnect incarnation ordering', () => { vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([detachedLease()] as ReturnType< typeof mockStore.getSshRemotePtyLeases >) - vi.mocked(mockStore.persistPtyBinding).mockImplementationOnce(() => { + // Step F: the durable write now happens inside the bind producer, so that is + // where the failure has to be injected. + vi.mocked(bindPaneShell).mockImplementationOnce(() => { throw new Error('disk full') }) const runtime = { onPtySpawned: vi.fn(), registerPty: vi.fn() } diff --git a/src/main/ssh/ssh-relay-session-recovery-races.test.ts b/src/main/ssh/ssh-relay-session-recovery-races.test.ts index 26dd798ac5a..f8853832ed1 100644 --- a/src/main/ssh/ssh-relay-session-recovery-races.test.ts +++ b/src/main/ssh/ssh-relay-session-recovery-races.test.ts @@ -705,7 +705,7 @@ describe('SshRelaySession recovery race fencing', () => { await session.reconnect(deps.mockConn) expect(attachForReconnectMock).toHaveBeenCalledTimes(2) - expect(attachForReconnectMock.mock.calls.at(-1)?.[2]).toMatchObject({ + expect(attachForReconnectMock.mock.calls.at(-1)?.[1]).toMatchObject({ status: 'checkpoint', deliveryToken: 'new-token', acceptedSourceEndSu: 8 @@ -763,7 +763,7 @@ describe('SshRelaySession recovery race fencing', () => { const replacementReconnect = session.reconnect(deps.mockConn) await Promise.all([staleReconnect, replacementReconnect]) - const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[2]) + const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[1]) expect(recoveryRequests).toHaveLength(2) expect(recoveryRequests[1]).toMatchObject({ status: 'checkpoint', diff --git a/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts b/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts index e0f1572d4d3..9e83768ca42 100644 --- a/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts @@ -54,6 +54,7 @@ type RejectedDeliverySession = { > rejectedPtyRecoveryAttempts: Map sourceIdentityByRelayPtyId: Map + watchMuxForRelayLoss: (mux: SshChannelMultiplexer) => void retireExitedPty: (payload: { id: string code: number @@ -89,9 +90,18 @@ function rejectedPayload(overrides: Partial = {}): SshPtyData function prepareSession() { const deps = createMockDeps() + const disposeHandlers: ((reason: 'shutdown' | 'connection_lost') => void)[] = [] const mux = { isDisposed: vi.fn(() => false), - dispose: vi.fn() + onDispose: vi.fn((handler: (reason: 'shutdown' | 'connection_lost') => void) => { + disposeHandlers.push(handler) + return () => {} + }), + dispose: vi.fn((reason: 'shutdown' | 'connection_lost' = 'shutdown') => { + for (const handler of disposeHandlers) { + handler(reason) + } + }) } as unknown as SshChannelMultiplexer const session = new SshRelaySession( 'target-1', @@ -101,6 +111,11 @@ function prepareSession() { ) const internals = session as unknown as RejectedDeliverySession internals.mux = mux + // Why wired: disposing this mux is what fires the host-wide relay-lost reconnect, so a test can + // observe the blast radius of a per-PTY escalation instead of only the local dispose call. + const relayLost = vi.fn() + session.setOnRelayLost(relayLost) + internals.watchMuxForRelayLoss(mux) internals.activePtyProviderGeneration = 23 internals.ptyConsumerSessionState = { mode: 'negotiated', @@ -110,7 +125,7 @@ function prepareSession() { ownerLease: 'owner-lease', outputFlowControl: { version: 1, windowSu: 64 } } - return { deps, internals, mux, session } + return { deps, internals, mux, relayLost, session } } describe('SshRelaySession rejected PTY delivery recovery', () => { @@ -231,7 +246,8 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { const recovery = reattachKnownPty.mock.calls[0]?.[0] expect(recovery?.ptyId).toBe('pty-bad') expect(Array.from(recovery?.activeLeaseByPtyId.keys() ?? [])).toEqual(['pty-bad']) - expect(Array.from(recovery?.expectedIdentityByPtyId.keys() ?? [])).toEqual([]) + // The sibling `expectedIdentityByPtyId` clause is gone with the map itself (STA-3077 step A): + // reattach no longer carries pane identity at all, which subsumes "carries none for this pty". expect(recovery?.targetedDeliveryRecovery).toBe('fresh-activation') expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [ 'pty-bad' @@ -243,8 +259,10 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { expect(mux.dispose).not.toHaveBeenCalled() }) + // Why this one still drops the channel: the relay refused to confirm the delivery was canceled, so + // no per-PTY fence is available — it is an unprovable state, not an exhausted retry count. it('reconnects instead of canceling an unprovable malformed delivery', async () => { - const { internals, mux } = prepareSession() + const { internals, mux, relayLost } = prepareSession() const reattach = vi.fn().mockResolvedValue(true) internals.reattachRejectedPty = reattach vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -259,6 +277,7 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { ) expect(mux.dispose).toHaveBeenCalledWith('connection_lost') + expect(relayLost).toHaveBeenCalledWith('target-1') expect(reattach).not.toHaveBeenCalled() }) @@ -338,7 +357,8 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { internals.reattachRejectedPty('pty-bad', mux, 23, 'confirm-existing') ).resolves.toBe(true) - expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', undefined, checkpoint) + // Two args since step A dropped the expected-identity parameter from the attach call. + expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', checkpoint) expect(commit).toHaveBeenCalledOnce() expect(mux.dispose).not.toHaveBeenCalled() }) @@ -357,11 +377,11 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { expect(internals.rejectedPtyRecoveryAttempts).toHaveLength(0) }) - // Why a channel drop rather than a terminal relay error: a terminal error clears the reconnect - // backoff and rotates provider authority, aborting every fs and git request on the target, so one - // PTY's undeliverable output would strand the whole connection in manual recovery. - it('bounds failed targeted recovery and escalates to a recoverable relay reconnect', async () => { - const { internals, mux, session } = prepareSession() + // Why this is the oracle: an exhausted retry budget is not proof of anything, and the relay channel + // is shared — dropping it rotates provider authority, aborts every in-flight fs and git request on + // the target and stalls every sibling PTY over one PTY's undeliverable output. + it('parks one PTY after its recovery budget runs out and leaves the shared channel and a sibling alone', async () => { + const { internals, mux, relayLost, session } = prepareSession() const reattach = vi.fn().mockResolvedValue(false) internals.reattachRejectedPty = reattach getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider) @@ -370,19 +390,90 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) await internals.acceptPtyData(rejectedPayload()) - await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 }) - - expect(reattach).toHaveBeenCalledTimes(2) - expect(reattach.mock.calls).toEqual([ - ['pty-bad', mux, 23, 'confirm-existing'], - ['pty-bad', mux, 23, 'confirm-existing'] - ]) - expect(mux.dispose).toHaveBeenCalledWith('connection_lost') - expect(warn).toHaveBeenCalledWith( - expect.stringContaining('PTY pty-bad delivery recovery exhausted') + await vi.waitFor( + () => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('PTY pty-bad delivery recovery exhausted') + ), + { timeout: 10_000 } ) + // Why the extra wait: a dispose would be scheduled off the same recovery pass as the warning. + await new Promise((resolve) => setTimeout(resolve, 400)) + + expect(mux.dispose).not.toHaveBeenCalled() + expect(relayLost).not.toHaveBeenCalled() expect(onTerminalError).not.toHaveBeenCalled() + expect(reattach).toHaveBeenCalledTimes(12) expect(acceptOutputDataMock).not.toHaveBeenCalled() + + await internals.acceptPtyData( + rejectedPayload({ + id: 'ssh:target-1@@pty-healthy', + data: 'healthy', + ptyIncarnation: 'incarnation-healthy', + source: source({ + relayPtyId: 'pty-healthy', + spanId: 'token-healthy:0:7', + deliveryToken: 'token-healthy', + sourceEndSu: 7 + }), + sourceRejected: undefined + }) + ) + + expect(acceptOutputDataMock).toHaveBeenCalledOnce() + expect(acceptOutputDataMock).toHaveBeenCalledWith( + expect.objectContaining({ id: 'ssh:target-1@@pty-healthy', data: 'healthy' }) + ) + }) + + // The clause above pins the containment; this one pins that containment is not abandonment. The + // park's only stated escape is the next relay open, and a shared channel that stays healthy never + // gives it one — so a pane could sit with no output and no way back, which is the exact state + // this change exists to prevent. The park has to expire. + it('re-arms a parked PTY once its cooldown has passed rather than leaving it dark', async () => { + const { internals } = prepareSession() + const reattach = vi.fn().mockResolvedValue(false) + internals.reattachRejectedPty = reattach + getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await internals.acceptPtyData(rejectedPayload()) + await vi.waitFor( + () => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('PTY pty-bad delivery recovery exhausted') + ), + { timeout: 10_000 } + ) + expect(reattach).toHaveBeenCalledTimes(12) + + // Let the exhaustion loop go quiet first: its last retries are timer-driven and would + // otherwise land after the clear below and satisfy the final clause on their own. + await new Promise((resolve) => setTimeout(resolve, 1500)) + reattach.mockClear() + + // Every frame below reuses the ORIGINAL delivery token, because that is what actually happens: + // retirement is ours, not the host's, and a stalled source keeps publishing the token we + // retired. It also matters that they are identical — retirement holds ONE key per relay PTY, + // so slipping a different token in here would overwrite that key, un-retire the original and + // let the final clause pass with the wake-up path still broken. + + // Before the cooldown: parked, so nothing retries. + await internals.acceptPtyData(rejectedPayload()) + await new Promise((resolve) => setTimeout(resolve, 600)) + expect(reattach).not.toHaveBeenCalled() + + // Backdate the park instead of burning a minute of real time — the property under test is + // elapsed time, not which timer implementation measures it. + const parked = internals.rejectedPtyRecoveryAttempts.get('ssh:target-1@@pty-bad') as { + parkedAt: number + } + parked.parkedAt -= 60_000 + + await internals.acceptPtyData(rejectedPayload()) + + await vi.waitFor(() => expect(reattach).toHaveBeenCalled(), { timeout: 10_000 }) }) it('stops without an error when the rejected PTY exited during recovery', async () => { @@ -404,6 +495,30 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { expect(mux.dispose).not.toHaveBeenCalled() }) + // The counterpart to the clause above, and the distinction the whole design rests on. hasPty is + // three-state: null means the provider has not listed the host yet, which is precisely the state + // a fresh provider is in right after the reconnect that produces rejected frames. Reading that + // as death drops the attempt and schedules nothing, and the delivery token is already retired, + // so nothing can revive it — the pane stays dark. Only an explicit false is proof. + it('keeps retrying while the PTY liveness is merely unknown', async () => { + const { internals, mux, session } = prepareSession() + const reattach = vi.fn().mockResolvedValue(false) + internals.reattachRejectedPty = reattach + getSshPtyProviderMock.mockReturnValue({ hasPty: () => null } as unknown as SshPtyProvider) + const onTerminalError = vi.fn() + session.setOnTerminalRelayError(onTerminalError) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await internals.acceptPtyData(rejectedPayload()) + + // More than one attempt is the whole point: unknown liveness must not end recovery. + await vi.waitFor(() => expect(reattach.mock.calls.length).toBeGreaterThan(1), { + timeout: 10_000 + }) + expect(onTerminalError).not.toHaveBeenCalled() + expect(mux.dispose).not.toHaveBeenCalled() + }) + // Why bounded: the bulk reconnect path caps the identical attach work at 8, and a relay that // starts rejecting across every PTY at once would otherwise open one attach round trip per PTY. it('caps concurrent targeted reattaches and coalesces repeats for one PTY', async () => { @@ -452,18 +567,18 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { }) it('does not let accepted frames refill the recovery budget indefinitely', async () => { - const { internals, mux, session } = prepareSession() + const { internals, mux, relayLost, session } = prepareSession() const reattach = vi.fn().mockResolvedValue(true) internals.reattachRejectedPty = reattach getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider) const onTerminalError = vi.fn() session.setOnTerminalRelayError(onTerminalError) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) // Why alternating, with a fresh bad token each round: every rejection retires its own delivery, // so a flapping PTY only keeps asking for recovery by moving onto new ones, and the accepted - // frame in between is what used to clear the budget outright. Each reattach here succeeds, so - // the consecutive budget is cleared legitimately too — only the per-generation ceiling can stop - // it. + // frame in between is what used to clear the budget outright — only the per-generation ceiling + // can stop it. for (let round = 0; round < 40; round++) { await internals.acceptPtyData( rejectedPayload({ @@ -487,9 +602,17 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { ) await Promise.resolve() } - await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 }) + await vi.waitFor( + () => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('PTY pty-bad delivery recovery exhausted') + ), + { timeout: 2000 } + ) expect(onTerminalError).not.toHaveBeenCalled() + expect(mux.dispose).not.toHaveBeenCalled() + expect(relayLost).not.toHaveBeenCalled() expect(acceptOutputDataMock).toHaveBeenCalledTimes(40) expect(reattach.mock.calls.length).toBeLessThanOrEqual(12) }) diff --git a/src/main/ssh/ssh-relay-session-terminal-error.test.ts b/src/main/ssh/ssh-relay-session-terminal-error.test.ts index a0f855543ef..026f64aafeb 100644 --- a/src/main/ssh/ssh-relay-session-terminal-error.test.ts +++ b/src/main/ssh/ssh-relay-session-terminal-error.test.ts @@ -104,6 +104,7 @@ function createMockDeps(): { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), diff --git a/src/main/ssh/ssh-relay-session-test-fixtures.ts b/src/main/ssh/ssh-relay-session-test-fixtures.ts index c5f084a8785..6fd72afd5a5 100644 --- a/src/main/ssh/ssh-relay-session-test-fixtures.ts +++ b/src/main/ssh/ssh-relay-session-test-fixtures.ts @@ -21,12 +21,14 @@ export function createMockDeps(): SshRelaySessionTestDeps { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), markSshRemotePtyLeasesForShutdown: vi.fn(), markSshRemotePtyLeasesAttachedAsync: vi.fn(), - persistPtyBinding: vi.fn() + markSshRemotePtyLeasesTerminatedAsync: vi.fn(), + persistPtyBinding: vi.fn().mockReturnValue(true) } as unknown as Store const mockPortForward = { removeAllForwards: vi.fn() diff --git a/src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts b/src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts new file mode 100644 index 00000000000..6ff4cf9f18d --- /dev/null +++ b/src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts @@ -0,0 +1,402 @@ +/** + * Journey 4 — two distinct final-host connections live at the same time, and one + * host's delivery failure may not reach the other host or a sibling PTY. + * + * Everything below SSH is real: two `SshRelaySession`s, two real + * `SshChannelMultiplexer`s, two real `SshPtyProvider`s, and two real + * `RelayDispatcher`s speaking the relay wire over an in-process pipe. The fault + * is injected the way a desynced relay actually produces it — a `pty.data` + * frame whose source header does not parse, with a delivery identity the host + * cannot cancel — so the whole rejection/recovery machinery runs for real. + * + * The oracle is the blast radius of exhausting ONE PTY's delivery-recovery + * budget: the shared relay channel, the sibling PTY on the same host, and the + * entire second host must be untouched. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { Store } from '../persistence' +import type { SshPortForwardManager } from './ssh-port-forward' +import type { SshConnection } from './ssh-connection' +import type { MultiplexerTransport } from './ssh-channel-multiplexer' +import { RelayDispatcher } from '../../relay/dispatcher' +import { getSshPtyProvider, unregisterSshPtyProvider } from '../ipc/pty' +import type { SshPtyProvider } from '../providers/ssh-pty-provider' +import { toAppSshPtyId } from '../providers/ssh-pty-id' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import { + installSshPtyOutputIntake, + type acceptSshPtyOutputData +} from '../ipc/ssh-pty-output-intake-registry' +import type { SshPtyOutputIntake } from '../ipc/ssh-pty-output-intake' + +vi.mock('./ssh-relay-deploy', () => ({ + deployAndLaunchRelay: vi.fn() +})) + +const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') +const { SshRelaySession } = await import('./ssh-relay-session') + +type DeliveredFrame = Parameters[0] + +/** The relay-side half of one host: a real dispatcher plus wire-level controls. */ +type FakeRelayHost = { + transport: MultiplexerTransport + dispatcher: RelayDispatcher + /** Publishes output on the PTY's live delivery, advancing its source cursor. */ + publish: (relayPtyId: string, data: string) => void + /** + * Publishes a frame whose source header does not parse (`sourceLengthSu` + * disagrees with the payload) but which still carries a complete delivery + * identity this client never installed. That combination is what routes the + * rejection into the bounded per-PTY recovery budget rather than an immediate + * channel reconnect. + */ + publishMalformed: (relayPtyId: string, data: string) => void + requestCounts: Map + /** When true, `pty.attach` is refused — the relay cannot re-prove the delivery. */ + refuseAttach: { value: boolean } + dispose: () => void +} + +function createFakeRelayHost(): FakeRelayHost { + const clientDataCallbacks: ((data: Buffer) => void)[] = [] + const clientCloseCallbacks: (() => void)[] = [] + const requestCounts = new Map() + const refuseAttach = { value: false } + const deliveries = new Map< + string, + { deliveryToken: string; ptyIncarnation: string; sentSu: number } + >() + let spawnCount = 0 + + const transport: MultiplexerTransport = { + write: (data) => { + setImmediate(() => dispatcher.feed(data)) + }, + onData: (cb) => { + clientDataCallbacks.push(cb) + }, + onClose: (cb) => { + clientCloseCallbacks.push(cb) + }, + close: () => { + for (const cb of clientCloseCallbacks) { + cb() + } + } + } + + const dispatcher = new RelayDispatcher((data) => { + setImmediate(() => { + for (const cb of clientDataCallbacks) { + cb(data) + } + }) + }) + + const count = (method: string): void => { + requestCounts.set(method, (requestCounts.get(method) ?? 0) + 1) + } + + dispatcher.onRequest('pty.openClient', async (params) => { + count('pty.openClient') + return { + protocolVersion: 1, + serverBuildId: 'test-relay-build', + clientGeneration: 1, + role: 'session-owner', + ownerGeneration: 1, + ownerLease: 'test-owner-lease', + resumed: params.resume !== undefined, + capabilities: { + outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } + } + } + }) + dispatcher.onRequest('session.resolveHome', async (params) => ({ + resolvedPath: params.path === '~' ? '/home/orca' : params.path + })) + dispatcher.onRequest('git.listWorktrees', async () => []) + dispatcher.onRequest('ports.detect', async () => ({ ports: [], platform: 'linux' })) + dispatcher.onRequest('pty.spawn', async () => { + count('pty.spawn') + spawnCount += 1 + const id = `remote-pty-${spawnCount}` + const ptyIncarnation = `incarnation-${id}` + const deliveryToken = `delivery-${id}` + deliveries.set(id, { deliveryToken, ptyIncarnation, sentSu: 0 }) + return { + id, + incarnationId: ptyIncarnation, + sourceActivation: { + status: 'pending', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation, + deliveryToken, + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + } + } + }) + dispatcher.onRequest('pty.getCwd', async () => { + count('pty.getCwd') + return '/home/orca/project' + }) + // Why `canceled: false`: the host cannot retire a delivery identity it never + // issued. That is the branch that asks the client to confirm the existing + // delivery rather than to drop the channel, so recovery enters the bounded + // retry budget this test is about. + dispatcher.onRequest('pty.cancelDelivery', async () => { + count('pty.cancelDelivery') + return { canceled: false } + }) + dispatcher.onRequest('pty.attach', async () => { + count('pty.attach') + if (refuseAttach.value) { + // Why not a not-found error: not-found is positive proof the PTY is gone + // and takes the destructive path. A refusal is the "unknown" case, which + // is what keeps the recovery budget burning down. + throw new Error('relay_attach_refused') + } + return { id: 'unused' } + }) + + const delivery = (relayPtyId: string) => { + const state = deliveries.get(relayPtyId) + if (!state) { + throw new Error(`relay has no delivery for ${relayPtyId}`) + } + return state + } + + return { + transport, + dispatcher, + publish: (relayPtyId, data) => { + const state = delivery(relayPtyId) + state.sentSu += data.length + dispatcher.notify('pty.data', { + id: relayPtyId, + data, + deliveryToken: state.deliveryToken, + ptyIncarnation: state.ptyIncarnation, + clientGeneration: 1, + ownerGeneration: 1, + sourceEndSu: state.sentSu, + sourceLengthSu: data.length + }) + }, + publishMalformed: (relayPtyId, data) => { + dispatcher.notify('pty.data', { + id: relayPtyId, + data, + deliveryToken: 'delivery-this-client-never-installed', + ptyIncarnation: 'incarnation-unknown', + clientGeneration: 1, + ownerGeneration: 1, + sourceEndSu: data.length + 1, + sourceLengthSu: data.length + 1 + }) + }, + requestCounts, + refuseAttach, + dispose: () => dispatcher.dispose() + } +} + +function createStore(): Store { + return { + getRepos: vi.fn().mockReturnValue([]), + getSshPtyConsumerRecovery: vi.fn().mockReturnValue(null), + upsertSshPtyConsumerRecovery: vi.fn(), + removeSshPtyConsumerRecovery: vi.fn(), + getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), + markSshRemotePtyLease: vi.fn(), + markSshRemotePtyLeases: vi.fn(), + markSshRemotePtyLeasesAsync: vi.fn(), + markSshRemotePtyLeasesForShutdown: vi.fn(), + markSshRemotePtyLeasesAttachedAsync: vi.fn(), + persistPtyBinding: vi.fn().mockReturnValue(true) + } as unknown as Store +} + +type LiveHost = { + targetId: string + relay: FakeRelayHost + session: InstanceType + provider: SshPtyProvider + relayLost: ReturnType + terminalError: ReturnType +} + +async function openHost(targetId: string): Promise { + const relay = createFakeRelayHost() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: relay.transport, + serverBuildId: 'test-relay-build', + platform: 'linux-x64' + }) + const session = new SshRelaySession( + targetId, + vi.fn().mockReturnValue({ + isDestroyed: () => false, + isVisible: () => true, + isMinimized: () => false, + webContents: { send: vi.fn() } + }), + createStore(), + { removeAllForwards: vi.fn().mockResolvedValue(undefined) } as unknown as SshPortForwardManager + ) + const relayLost = vi.fn() + const terminalError = vi.fn() + session.setOnRelayLost(relayLost) + session.setOnTerminalRelayError(terminalError) + await session.establish({} as SshConnection) + const provider = getSshPtyProvider(targetId) as SshPtyProvider | undefined + if (!provider) { + throw new Error(`no PTY provider registered for ${targetId}`) + } + return { targetId, relay, session, provider, relayLost, terminalError } +} + +describe('two live SSH hosts over real relay dispatchers', () => { + const hosts: LiveHost[] = [] + let delivered: DeliveredFrame[] = [] + let uninstallIntake: (() => void) | null = null + let warnSpy: ReturnType + + beforeEach(() => { + vi.clearAllMocks() + delivered = [] + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + // Why the real registration seam: the session publishes accepted output + // through `acceptSshPtyOutputData`, so installing a recording intake + // observes the production delivery boundary without stubbing the module. + uninstallIntake = installSshPtyOutputIntake({ + acceptData: async (event) => { + delivered.push(event) + return { status: 'accepted' } as never + }, + acceptExit: async () => {}, + closeGeneration: () => {}, + dispose: () => {} + } as unknown as SshPtyOutputIntake) + }) + + afterEach(() => { + for (const host of hosts.splice(0)) { + host.session.dispose() + host.relay.dispose() + unregisterSshPtyProvider(host.targetId) + } + uninstallIntake?.() + uninstallIntake = null + warnSpy.mockRestore() + }) + + /** + * Brings both hosts up, gives host A two PTYs and host B one, proves all three + * deliver, then burns host A's PTY-1 delivery-recovery budget to exhaustion. + * + * The two clauses are separate tests on purpose: a mutation that only breaks + * same-host containment must be seen to leave the cross-host clause green, + * which proves the clauses independently rather than jointly. + */ + async function exhaustOnePtyOnHostA(): Promise<{ + hostA: LiveHost + hostB: LiveHost + a2: { id: string } + b1: { id: string } + }> { + const hostA = await openHost('host-a') + const hostB = await openHost('host-b') + hosts.push(hostA, hostB) + + const a1 = await hostA.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' }) + const a2 = await hostA.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' }) + const b1 = await hostB.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' }) + expect(a1.id).toBe(toAppSshPtyId('host-a', 'remote-pty-1')) + expect(a2.id).toBe(toAppSshPtyId('host-a', 'remote-pty-2')) + expect(b1.id).toBe(toAppSshPtyId('host-b', 'remote-pty-1')) + // Two distinct final-host connections, each with its own provider identity. + expect(hostA.provider).not.toBe(hostB.provider) + + hostA.relay.publish('remote-pty-1', 'a1-before') + hostA.relay.publish('remote-pty-2', 'a2-before') + hostB.relay.publish('remote-pty-1', 'b1-before') + await vi.waitFor(() => expect(delivered).toHaveLength(3), { timeout: 5_000 }) + delivered = [] + + // The fault: one PTY on host A can never re-prove its delivery. + hostA.relay.refuseAttach.value = true + hostA.relay.publishMalformed('remote-pty-1', 'a1-poisoned') + await vi.waitFor( + () => + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('remote-pty-1 delivery recovery exhausted') + ), + { timeout: 60_000 } + ) + // Why the extra settle: a channel drop is scheduled off the same recovery + // pass that logs the exhaustion, not before it. + await new Promise((resolve) => setTimeout(resolve, 500)) + + // The budget was really spent on relay round trips, so the fault is live. + expect(hostA.relay.requestCounts.get('pty.attach') ?? 0).toBeGreaterThanOrEqual(12) + expect(hostA.relay.requestCounts.get('pty.cancelDelivery') ?? 0).toBeGreaterThan(0) + // The undeliverable frame itself never reached a consumer. + expect(delivered.some((frame) => frame.data === 'a1-poisoned')).toBe(false) + + return { hostA, hostB, a2, b1 } + } + + it('leaves the sibling PTY and the shared channel on the same host working', async () => { + const { hostA, a2 } = await exhaustOnePtyOnHostA() + + hostA.relay.publish('remote-pty-2', 'a2-after') + await vi.waitFor( + () => + expect( + delivered.filter((frame) => frame.id === a2.id && frame.data === 'a2-after'), + 'the sibling PTY stopped receiving output after its neighbour was parked' + ).toHaveLength(1), + { timeout: 10_000 } + ) + // A live request round trip: the shared channel still carries the fs/git + // control plane that dropping it would have aborted. + await expect(hostA.provider.getCwd(a2.id)).resolves.toBe('/home/orca/project') + expect(hostA.relayLost, 'host A lost its relay channel over one PTY').not.toHaveBeenCalled() + expect(hostA.terminalError, 'host A was parked in manual recovery').not.toHaveBeenCalled() + }, 120_000) + + it('leaves the second host entirely untouched', async () => { + const { hostB, b1 } = await exhaustOnePtyOnHostA() + + expect(hostB.relayLost, 'host B lost its relay channel').not.toHaveBeenCalled() + expect(hostB.terminalError, 'host B was parked in manual recovery').not.toHaveBeenCalled() + expect( + hostB.relay.requestCounts.get('pty.attach') ?? 0, + "host A's recovery reached host B's relay" + ).toBe(0) + expect( + hostB.relay.requestCounts.get('pty.cancelDelivery') ?? 0, + "host A's delivery cancellation reached host B's relay" + ).toBe(0) + expect( + hostB.relay.requestCounts.get('pty.openClient') ?? 0, + 'host B renegotiated its consumer session' + ).toBe(1) + await expect(hostB.provider.getCwd(b1.id)).resolves.toBe('/home/orca/project') + hostB.relay.publish('remote-pty-1', 'b1-after') + await vi.waitFor( + () => + expect( + delivered.filter((frame) => frame.id === b1.id && frame.data === 'b1-after') + ).toHaveLength(1), + { timeout: 10_000 } + ) + }, 120_000) +}) diff --git a/src/main/ssh/ssh-relay-session.test.ts b/src/main/ssh/ssh-relay-session.test.ts index 570fca21fd4..8563c552aaf 100644 --- a/src/main/ssh/ssh-relay-session.test.ts +++ b/src/main/ssh/ssh-relay-session.test.ts @@ -531,26 +531,8 @@ describe('SshRelaySession', () => { ) }) - it('forwards a lease tab identity to reattach so a reset relay cannot cross-wire it', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() - const { getSshPtyProvider } = await import('../ipc/pty') - const mockAttach = vi.fn().mockResolvedValue(undefined) - vi.mocked(getSshPtyProvider).mockReturnValue({ - attachForReconnect: mockAttach, - dispose: vi.fn() - } as unknown as ReturnType) - vi.mocked(getPtyIdsForConnection).mockReturnValue([]) - vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([ - { targetId: 'target-1', ptyId: 'pty-1', state: 'detached', tabId: 'tab-a' } - ] as ReturnType) - - const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) - await session.establish(mockConn) - - expect(mockAttach).toHaveBeenCalledWith('pty-1', { tabId: 'tab-a' }) - }) - - it('forwards a lease pane identity when leaf identity is available', async () => { + // New relays prioritize incarnation; old relays ignore that additive field and need this fence. + it('preserves pane identity for relays that predate incarnation fencing', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const { getSshPtyProvider } = await import('../ipc/pty') const mockAttach = vi.fn().mockResolvedValue(undefined) @@ -567,7 +549,40 @@ describe('SshRelaySession', () => { const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) - expect(mockAttach).toHaveBeenCalledWith('pty-1', { + expect(mockAttach).toHaveBeenCalledWith('pty-1', undefined, undefined, { + paneKey: `tab-a:${leafId}`, + tabId: 'tab-a' + }) + }) + + // This is the path that reattaches every known pty when a relay comes back — precisely when ids + // have been reissued from pty-1 — so it is the one that most needs to say which shell it means. + // The shell's own identity is authoritative; pane identity remains the fallback for old relays. + it('forwards the shell identity its lease recorded when reattaching', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const { getSshPtyProvider } = await import('../ipc/pty') + const mockAttach = vi.fn().mockResolvedValue(undefined) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + const leafId = '11111111-1111-4111-8111-111111111111' + vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([ + { + targetId: 'target-1', + ptyId: 'pty-1', + state: 'detached', + tabId: 'tab-a', + leafId, + incarnationId: 'inc-host-1' + } + ] as ReturnType) + + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + + expect(mockAttach).toHaveBeenCalledWith('pty-1', undefined, 'inc-host-1', { paneKey: `tab-a:${leafId}`, tabId: 'tab-a' }) @@ -602,13 +617,10 @@ describe('SshRelaySession', () => { await session.reconnect(mockConn) - expect(mockAttach).toHaveBeenCalledWith('pty-1', { - paneKey: `tab-old:${staleLeafId}`, - tabId: 'tab-old' - }) expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-1') expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-1') expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith('target-1', 'pty-1', 'expired') + expect(mockAttach).toHaveBeenCalledOnce() expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', { id: 'ssh:target-1@@pty-1', code: -1 @@ -671,7 +683,11 @@ describe('SshRelaySession', () => { expect(mockStore.markSshRemotePtyLeasesAttachedAsync).not.toHaveBeenCalled() }) - it('invalidates and broadcasts remote PTYs that cannot reattach after relay reconnect', async () => { + // INVERTED for STA-3077 step E-0. This case previously asserted that a PTY the relay reports as + // not-found is invalidated and broadcast as an exit. A not-found is not proof of death, so the + // broadcast was a fabricated one; what must still hold is that the healthy sibling reattaches and + // the unproven one is left alone rather than written off. + it('leaves an unreattachable remote PTY alone while its sibling reattaches', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) @@ -693,12 +709,9 @@ describe('SshRelaySession', () => { expect(mockAttach).toHaveBeenCalledWith('pty-stale') expect(mockAttach).toHaveBeenCalledWith('pty-live') - expect(clearProviderPtyState).toHaveBeenCalledWith('ssh:target-1@@pty-stale') - expect(deletePtyOwnership).toHaveBeenCalledWith('ssh:target-1@@pty-stale') - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { - id: 'ssh:target-1@@pty-stale', - code: -1 - }) + expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-stale') + expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-stale') + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) }) it('retries transient reattach failure without tearing down provider registration', async () => { diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index 820eb7e1360..ea56dd88786 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -12,6 +12,7 @@ import type { SshPtyAttachResult } from '../providers/ssh-pty-session-reattach' import type { SshPtyDataCallback, SshPtyExitCallback } from '../providers/ssh-pty-provider-contract' import type { SshPtyRecoveryActivationLease } from '../providers/ssh-pty-notification-routing' import { isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors' +import { parseMatchingPtyExitedError } from '../../shared/ssh-pty-failure-tokens' import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' import { SshFilesystemProvider } from '../providers/ssh-filesystem-provider' import { isMethodNotFoundError } from './ssh-filesystem-stream-reader' @@ -32,6 +33,8 @@ import { import { _internals as openCodeInternals } from '../opencode/hook-service' import { getPiAgentStatusExtensionSource } from '../pi/agent-status-extension-source' import { + bindPaneShell, + resolvePaneShellTabId, registerSshPtyProvider, unregisterSshPtyProvider, getSshPtyProvider, @@ -95,6 +98,8 @@ import { parseRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output' import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' +import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' +import { isValidTerminalTabId } from '../../shared/terminal-tab-id' import { SSH_AI_VAULT_LIST_SESSIONS_METHOD, SSH_AI_VAULT_LIST_SESSIONS_TIMEOUT_MS, @@ -103,8 +108,6 @@ import { type SshAiVaultRelayListParams, type SshAiVaultRelayTitleParams } from '../../shared/ssh-ai-vault-relay' -import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' -import { isValidTerminalTabId } from '../../shared/terminal-tab-id' import { openSshPtyConsumerSession, type OpenSshPtyConsumerSessionOptions, @@ -138,11 +141,15 @@ const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8 const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000 const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50 const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200 -const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2 -// Why a second ceiling: the consecutive budget resets whenever a reattach succeeds, so a PTY that -// alternates recovered and rejected frames would otherwise reattach forever — each one costs a -// store read, an attach round trip and a store write. +// Why a ceiling at all: a PTY that alternates recovered and rejected frames would otherwise reattach +// forever — each one costs a store read, an attach round trip and a store write. Exhausting it parks +// this PTY's delivery; it never authorizes tearing anything down. const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12 +// Why a park expires: exhaustion parks one PTY's delivery instead of dropping the shared channel, +// and the escape it names — the next relay open — may never come while that channel stays healthy. +// A pane with no output and no way back is the state this change exists to prevent, so the park is +// a cooldown rather than a verdict: recovery is rate-limited, never abandoned. +const SSH_REJECTED_PTY_RECOVERY_PARK_MS = 60_000 const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150 const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed' @@ -191,15 +198,15 @@ type RemoteCliBridgeEnv = { pathDelimiter?: ':' | ';' } -type ExpectedPtyIdentity = { paneKey?: string; tabId?: string } +type LegacyExpectedPtyIdentity = { paneKey?: string; tabId?: string } type TargetedDeliveryRecovery = 'confirm-existing' | 'fresh-activation' -function expectedIdentityForLease(lease: { +function legacyExpectedIdentityForLease(lease: { tabId?: string leafId?: string -}): ExpectedPtyIdentity | null { +}): LegacyExpectedPtyIdentity | undefined { if (typeof lease.tabId !== 'string' || lease.tabId.length === 0) { - return null + return undefined } const paneKey = isValidTerminalTabId(lease.tabId) && @@ -207,10 +214,7 @@ function expectedIdentityForLease(lease: { isTerminalLeafId(lease.leafId) ? makePaneKey(lease.tabId, lease.leafId) : undefined - return { - ...(paneKey ? { paneKey } : {}), - tabId: lease.tabId - } + return { ...(paneKey ? { paneKey } : {}), tabId: lease.tabId } } function parseRecoveryComplete(params: Record): PtySourceRecoveryComplete | null { @@ -343,9 +347,10 @@ export class SshRelaySession { string, { providerGeneration: number - attempts: number generationAttempts: number reported: boolean + /** When the budget ran out; 0 while it has not. Drives the park cooldown. */ + parkedAt: number } >() private readonly rejectedPtyRecoveryRetries = new Set>() @@ -1789,7 +1794,8 @@ export class SshRelaySession { const offeredSource = payload.source if ( offeredSource && - this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource) + this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource) && + !this.isParkedRecoveryDue(payload.id) ) { return Promise.resolve() } @@ -1846,6 +1852,19 @@ export class SshRelaySession { }) } + /** + * A parked PTY whose cooldown has elapsed must be let back past the retired-delivery filter. + * Retirement is ours, not the host's: a stalled source keeps publishing the very token we + * retired, so that filter would swallow every frame that could re-arm the park and the pane + * would stay dark for good — the failure the cooldown exists to rule out. Letting one through + * costs nothing: it is re-classified as rejected and re-retired, so no output reaches the + * terminal; it only regains the ability to ask for recovery. + */ + private isParkedRecoveryDue(appPtyId: string): boolean { + const attempt = this.rejectedPtyRecoveryAttempts.get(appPtyId) + return !!attempt?.parkedAt && Date.now() - attempt.parkedAt >= SSH_REJECTED_PTY_RECOVERY_PARK_MS + } + private recoverRejectedPtyDelivery( payload: SshPtyDataPayload, source: SshPtyDataPayload['source'] @@ -1880,26 +1899,28 @@ export class SshRelaySession { const attempt = previous?.providerGeneration === providerGeneration ? previous - : { providerGeneration, attempts: 0, generationAttempts: 0, reported: false } - if ( - attempt.attempts >= SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS || - attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS - ) { - if (!attempt.reported) { - attempt.reported = true - console.warn( - `[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; dropping the relay channel to reconnect` - ) - // Why a channel drop and not a terminal relay error: a terminal error clears the reconnect - // backoff, rotates provider authority (aborting every in-flight fs and git request on the - // target) and parks the target in a manual-recovery state — over one PTY's delivery. Losing - // the channel is the recoverable escalation, and it is what this path did before targeted - // recovery existed. - mux.dispose('connection_lost') + : { providerGeneration, generationAttempts: 0, reported: false, parkedAt: 0 } + if (attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS) { + // Why park instead of dropping the relay channel: a retry count is not proof of anything, and + // the channel is shared — dropping it rotates provider authority, aborts every in-flight fs + // and git request on the target and stalls every sibling PTY over one PTY's delivery. The + // remote shell keeps running and its lease stands. + if (attempt.parkedAt && Date.now() - attempt.parkedAt >= SSH_REJECTED_PTY_RECOVERY_PARK_MS) { + // The park has served its cooldown. Re-arm rather than leave the pane dark for good. + attempt.generationAttempts = 0 + attempt.reported = false + attempt.parkedAt = 0 + } else { + if (!attempt.reported) { + attempt.reported = true + attempt.parkedAt = Date.now() + console.warn( + `[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; parking its delivery for ${SSH_REJECTED_PTY_RECOVERY_PARK_MS}ms or until the next relay open` + ) + } + return } - return } - attempt.attempts++ attempt.generationAttempts++ this.rejectedPtyRecoveryAttempts.set(appPtyId, attempt) void this.rejectedPtyReattaches @@ -1915,13 +1936,9 @@ export class SshRelaySession { ) .then( (recovered) => { - if (recovered) { - // Why only a completed reattach clears this: an accepted frame proves nothing about the - // delivery that was rejected, and resetting on one lets a flapping PTY reattach forever. - attempt.attempts = 0 - return + if (!recovered) { + this.retryRejectedPtyDelivery(payload, source, appPtyId) } - this.retryRejectedPtyDelivery(payload, source, appPtyId) }, (error: unknown) => { console.warn(`[ssh-relay-session] PTY ${relayPtyId} targeted delivery recovery failed`, { @@ -1935,14 +1952,23 @@ export class SshRelaySession { // Why liveness is checked before retrying: reattachKnownPty resolves without claiming the lease // when the PTY exited mid-attach, which is indistinguishable from a failed reattach at the call - // site. Retrying that race twice would drop the relay channel over an ordinary PTY exit. + // site, so an ordinary exit would otherwise burn the whole recovery budget on attach round trips. private retryRejectedPtyDelivery( payload: SshPtyDataPayload, source: SshPtyDataPayload['source'], appPtyId: string ): void { const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined - if (!ptyProvider || typeof ptyProvider.hasPty !== 'function' || !ptyProvider.hasPty(appPtyId)) { + // Why `=== false` and not `!`: hasPty is three-state, and null means the provider has not + // listed the host yet — ignorance, not death, which is exactly the state a fresh provider is + // in right after the reconnect that produces these rejected frames. Treating it as death + // dropped the recovery attempt and scheduled nothing, and since the delivery token is already + // retired, no later frame could revive it: the pane stayed dark. Only proof stops us. + if ( + !ptyProvider || + typeof ptyProvider.hasPty !== 'function' || + ptyProvider.hasPty(appPtyId) === false + ) { this.rejectedPtyRecoveryAttempts.delete(appPtyId) return } @@ -1979,21 +2005,25 @@ export class SshRelaySession { const activeLeaseByPtyId = activeLease ? new Map([[relayPtyId, activeLease]]) : new Map() - const expectedIdentity = activeLease ? expectedIdentityForLease(activeLease) : undefined const attachedLeaseIds = new Set() + const terminatedLeaseIds = new Set() await this.reattachKnownPty({ ptyProvider, ptyId: relayPtyId, activeLeaseByPtyId, - expectedIdentityByPtyId: expectedIdentity - ? new Map([[relayPtyId, expectedIdentity]]) - : new Map(), attachedLeaseIds, + terminatedLeaseIds, mux, providerGeneration, shouldContinue, targetedDeliveryRecovery }) + if (terminatedLeaseIds.size > 0) { + await this.store.markSshRemotePtyLeasesTerminatedAsync( + this.targetId, + Array.from(terminatedLeaseIds) + ) + } if (attachedLeaseIds.size > 0 && shouldContinue()) { await this.store.markSshRemotePtyLeasesAttachedAsync( this.targetId, @@ -2245,21 +2275,21 @@ export class SshRelaySession { mux: SshChannelMultiplexer, shouldContinue: () => boolean ): Promise { + // Why: installs predating pane-keyed supersession carry duplicate live leases; + // fanning out over them is what grafted panes the user never opened (STA-3077). + const retired = await this.store.supersedeDuplicatePaneLeases(this.targetId) + if (retired > 0) { + console.info( + `[ssh-relay-session] Retired ${retired} duplicate pane lease(s) for ${this.targetId}; their remote shells are left running.` + ) + } const activeLeases = this.store .getSshRemotePtyLeases(this.targetId) .filter((lease) => lease.state !== 'terminated' && lease.state !== 'expired') const activeLeaseByPtyId = new Map(activeLeases.map((lease) => [lease.ptyId, lease])) const leasedPtyIds = activeLeases.map((lease) => lease.ptyId) - // Why: pass pane identity so the relay can reject cross-generation id collisions; tabId falls back for pre-leafId leases. - const expectedIdentityByPtyId = new Map( - activeLeases - .map((lease): [string, ExpectedPtyIdentity] | null => { - const expected = expectedIdentityForLease(lease) - return expected ? [lease.ptyId, expected] : null - }) - .filter((entry): entry is [string, ExpectedPtyIdentity] => entry !== null) - ) const attachedLeaseIds = new Set() + const terminatedLeaseIds = new Set() // Why: after app restart ptyOwnership is empty, but durable SSH leases still describe grace-window survivors. const ptyIds = Array.from( new Set([ @@ -2286,8 +2316,8 @@ export class SshRelaySession { ptyProvider, ptyId, activeLeaseByPtyId, - expectedIdentityByPtyId, attachedLeaseIds, + terminatedLeaseIds, mux, providerGeneration, shouldContinue @@ -2307,6 +2337,12 @@ export class SshRelaySession { await Promise.all( Array.from({ length: Math.min(SSH_PTY_REATTACH_MAX_CONCURRENCY, ptyIds.length) }, worker) ) + if (terminatedLeaseIds.size > 0) { + await this.store.markSshRemotePtyLeasesTerminatedAsync( + this.targetId, + Array.from(terminatedLeaseIds) + ) + } if (attachedLeaseIds.size > 0 && shouldContinue()) { await this.store.markSshRemotePtyLeasesAttachedAsync( this.targetId, @@ -2319,8 +2355,8 @@ export class SshRelaySession { ptyProvider: SshPtyProvider ptyId: string activeLeaseByPtyId: Map - expectedIdentityByPtyId: Map attachedLeaseIds: Set + terminatedLeaseIds: Set mux: SshChannelMultiplexer providerGeneration: number shouldContinue: () => boolean @@ -2330,8 +2366,8 @@ export class SshRelaySession { ptyProvider, ptyId, activeLeaseByPtyId, - expectedIdentityByPtyId, attachedLeaseIds, + terminatedLeaseIds, mux, providerGeneration, shouldContinue, @@ -2358,12 +2394,17 @@ export class SshRelaySession { targetedDeliveryRecovery === 'fresh-activation' ? undefined : await this.sourceRecoveryRequest(appPtyId) + // The shell this lease names, as the host attested it. Without it the main reconnect path + // attaches by id alone, and a replaced relay reissues ids from pty-1. const attachResult = await this.attachPtyWithRetry( ptyProvider, ptyId, - expectedIdentityByPtyId.get(ptyId), recoveryRequest, - shouldContinue + shouldContinue, + activeLeaseByPtyId.get(ptyId)?.incarnationId, + activeLeaseByPtyId.get(ptyId) + ? legacyExpectedIdentityForLease(activeLeaseByPtyId.get(ptyId)!) + : undefined ) sourceActivationLease = attachResult.sourceActivationLease if (!shouldContinue()) { @@ -2509,7 +2550,13 @@ export class SshRelaySession { if (!shouldContinue()) { return } - this.handlePtyReattachFailure(ptyId, appPtyId, pendingReattach, error) + this.handlePtyReattachFailure( + ptyId, + pendingReattach, + error, + activeLeaseByPtyId.get(ptyId)?.incarnationId, + terminatedLeaseIds + ) } finally { recoveryActivationLease?.retire() sourceActivationLease?.rollback() @@ -2548,22 +2595,42 @@ export class SshRelaySession { lease: SshPtyLease | undefined ): void { if (lease?.worktreeId && lease.tabId && lease.leafId) { - this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { - tabId: lease.tabId, - leafId: lease.leafId, - incarnationId - }) + // Why bind before registering: `mayCreate: false` refuses when the pane is + // gone, and registering first would surface a pane the user never opened. + // A thrown write is unknown, not a refusal, so it must not detach anything. + // The lease's tabId was frozen when it was written and the pane may have moved since, so the + // live layout outranks it. Resolved before the write, so a throw cannot lose it and leave the + // graph registered under the tab the pane left. + const tabId = resolvePaneShellTabId(this.store, lease.leafId) ?? lease.tabId + let bind: { bound: boolean; tabId: string } | null = null try { - this.store.persistPtyBinding({ + bind = bindPaneShell({ + store: this.store, worktreeId: lease.worktreeId, - tabId: lease.tabId, + tabId, leafId: lease.leafId, ptyId: appPtyId, - incarnationId + incarnationId, + mayCreate: false }) } catch (error) { console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error) } + if (bind?.bound === false) { + // Unresolved, not dead: the remote shell keeps running and stays + // reattachable once a durable pane names it again. + console.info( + `[ssh-relay-session] No durable pane owns ${appPtyId} on ${this.targetId}; left running unbound.` + ) + return + } + // The tab the bind resolved, not the lease's frozen one — a moved pane must not be recorded + // in the graph under the tab it left while its record and fence name the tab it is in. + this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { + tabId, + leafId: lease.leafId, + incarnationId + }) return } this.runtime?.onPtySpawned(appPtyId, incarnationId, { awaitsRegistration: false }) @@ -2572,9 +2639,10 @@ export class SshRelaySession { private async attachPtyWithRetry( ptyProvider: SshPtyProvider, ptyId: string, - expectedIdentity: ExpectedPtyIdentity | undefined, recoveryRequest: PtySourceRecoveryRequest | undefined, - shouldContinue: () => boolean + shouldContinue: () => boolean, + expectedIncarnationId: string | undefined, + legacyExpectedIdentity: LegacyExpectedPtyIdentity | undefined ): Promise { let lastError: unknown for (let attempt = 0; attempt < 2; attempt++) { @@ -2585,12 +2653,26 @@ export class SshRelaySession { return await this.attachPtyWithDeadline( ptyProvider, ptyId, - expectedIdentity, - recoveryRequest + recoveryRequest, + expectedIncarnationId, + legacyExpectedIdentity ) } catch (error) { lastError = error - if (!shouldContinue() || isSshPtyNotFoundError(error) || attempt === 1) { + // A proven exit is as final as an unknown id: retrying either only burns the wait before + // the same answer. Without this the liveness-probe reap, which now answers with proof + // rather than "not found", would no longer match and would cost a needless retry. + if ( + !shouldContinue() || + isSshPtyNotFoundError(error) || + isSshPtyIdentityMismatchError(error) || + parseMatchingPtyExitedError( + error instanceof Error ? error.message : String(error), + ptyId, + expectedIncarnationId + ) !== null || + attempt === 1 + ) { throw error } await this.waitForPtyReattachRetry() @@ -2602,8 +2684,9 @@ export class SshRelaySession { private async attachPtyWithDeadline( ptyProvider: SshPtyProvider, ptyId: string, - expectedIdentity: ExpectedPtyIdentity | undefined, - recoveryRequest: PtySourceRecoveryRequest | undefined + recoveryRequest: PtySourceRecoveryRequest | undefined, + expectedIncarnationId: string | undefined, + legacyExpectedIdentity: LegacyExpectedPtyIdentity | undefined ): Promise { let timer: ReturnType | undefined let timedOut = false @@ -2617,13 +2700,17 @@ export class SshRelaySession { timer.unref?.() }) try { - const attach = expectedIdentity - ? recoveryRequest - ? ptyProvider.attachForReconnect(ptyId, expectedIdentity, recoveryRequest) - : ptyProvider.attachForReconnect(ptyId, expectedIdentity) - : recoveryRequest - ? ptyProvider.attachForReconnect(ptyId, undefined, recoveryRequest) - : ptyProvider.attachForReconnect(ptyId) + const attach = + expectedIncarnationId !== undefined || legacyExpectedIdentity !== undefined + ? ptyProvider.attachForReconnect( + ptyId, + recoveryRequest, + expectedIncarnationId, + legacyExpectedIdentity + ) + : recoveryRequest !== undefined + ? ptyProvider.attachForReconnect(ptyId, recoveryRequest) + : ptyProvider.attachForReconnect(ptyId) const guardedAttach = attach.then((result) => { if (timedOut) { result.sourceActivationLease?.rollback() @@ -2650,40 +2737,31 @@ export class SshRelaySession { private handlePtyReattachFailure( ptyId: string, - appPtyId: string, pending: PendingPtyReattach, - error: unknown + error: unknown, + expectedIncarnationId: string | undefined, + terminatedLeaseIds: Set ): void { - if (!isSshPtyNotFoundError(error)) { - pending.restoreRequired = 'reattachAttemptsExhausted' - this.wakeRecovery(pending) - console.warn( - `[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after bounded reattach attempts failed: ${ - error instanceof Error ? error.message : String(error) - }` - ) - return - } - if (isSshPtyIdentityMismatchError(error)) { - console.warn( - `[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${ - error instanceof Error ? error.message : String(error) - }` - ) - return + // An exact exit proof settles the lease; queue it for one post-worker durable retirement batch. + // `terminated` is not the recovery-grant state, so this cannot authorize a replacement. + + const exitProof = parseMatchingPtyExitedError( + error instanceof Error ? error.message : String(error), + ptyId, + expectedIncarnationId + ) + if (exitProof) { + terminatedLeaseIds.add(ptyId) } + // Nothing else proves it — not even a not-found, which the relay also returns when it merely + // cannot hand this id back — so every other failure leaves the pane detached and recoverable. + pending.restoreRequired = 'reattachAttemptsExhausted' + this.wakeRecovery(pending) console.warn( - `[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${ + `[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after reattach failed: ${ error instanceof Error ? error.message : String(error) }` ) - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) - } } private async sourceRecoveryRequest( diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 0f90182d053..54e03a81f58 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -1469,6 +1469,8 @@ export type PreloadApi = { rows: number cwd?: string cwdFallback?: 'worktree' + /** Create a shell for this pane instead of adopting the one it records; see pty.ts. */ + createFreshShellForUnreachablePane?: boolean env?: Record envToDelete?: string[] command?: string diff --git a/src/preload/index.ts b/src/preload/index.ts index 58aec899f03..4808dd3b979 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -943,6 +943,7 @@ const api = { rows: number cwd?: string cwdFallback?: 'worktree' + createFreshShellForUnreachablePane?: boolean env?: Record envToDelete?: string[] command?: string diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 06eaa15d5ff..f45727c19db 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -48,7 +48,6 @@ import { IMMEDIATE_PTY_EXIT_TIMEOUT_MS, MAX_RELAY_PTY_SESSIONS, PtyHandler, - attachIdentityMismatches, formatNodePtyUnavailableMessage } from './pty-handler' import { RelayDispatcher } from './dispatcher' @@ -1386,10 +1385,10 @@ describe('PtyHandler', () => { } ) - it('reports not-found and reaps a reattach whose backing shell is dead', async () => { + it('proves the exit and reaps a reattach whose backing shell is dead', async () => { // Why: a lingering managed entry whose child died without an onExit would // otherwise attach-succeed with an empty replay and strand the pane on a - // black shell. A provably-dead pid must surface as not-found so the SSH + // black shell. A provably-dead pid must surface as a PROVEN exit so the SSH // provider maps it to SSH_SESSION_EXPIRED and the pane respawns fresh. let onExitCb: ((evt: { exitCode: number }) => void) | undefined mockPtySpawn.mockReturnValue({ @@ -1402,15 +1401,25 @@ describe('PtyHandler', () => { const exits: { id: string; paneKey?: string }[] = [] handler.setExitListener((evt) => exits.push(evt)) - await dispatcher.callRequest('pty.spawn', { env: { ORCA_PANE_KEY: 'tab-dead:0' } }) + const spawned = (await dispatcher.callRequest('pty.spawn', { + env: { ORCA_PANE_KEY: 'tab-dead:0' } + })) as { id: string; incarnationId: string } expect(handler.activePtyCount).toBe(1) expect(onExitCb).toBeDefined() const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) try { + // INVERTED: the reap is unchanged, but its answer is no longer the ordinary unknown. The + // relay held this shell and found its pid gone, which is first-hand proof of death — the + // one observation that separates a dead shell from one a replacement relay never knew. await expect( - dispatcher.callRequest('pty.attach', { id: 'pty-1', suppressReplayNotification: true }) - ).rejects.toThrow('PTY "pty-1" not found') + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) } finally { aliveSpy.mockRestore() } @@ -1419,9 +1428,16 @@ describe('PtyHandler', () => { // is freed so a later attach also cleanly reports not-found. expect(exits).toEqual([{ id: 'pty-1', paneKey: 'tab-dead:0' }]) expect(handler.activePtyCount).toBe(0) + // INVERTED: a LATER attach still answers, and now answers with what this process observed — + // the remembered exit outlives the reap, which is the whole point of remembering it. await expect( - dispatcher.callRequest('pty.attach', { id: 'pty-1', suppressReplayNotification: true }) - ).rejects.toThrow('PTY "pty-1" not found') + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) }) it.skipIf(process.platform === 'win32')( @@ -1476,7 +1492,10 @@ describe('PtyHandler', () => { onData: vi.fn(), onExit: vi.fn() }) - await dispatcher.callRequest('pty.spawn', {}) + const spawned = (await dispatcher.callRequest('pty.spawn', {})) as { + id: string + incarnationId: string + } let shutdown: Promise | undefined const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockImplementation(() => { @@ -1484,9 +1503,14 @@ describe('PtyHandler', () => { return false }) try { - await expect(dispatcher.callRequest('pty.attach', { id: 'pty-1' })).rejects.toThrow( - 'PTY "pty-1" not found' - ) + // INVERTED with the reap above: proof of exit, not an unknown id. + await expect( + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) await expect(shutdown).resolves.toBeUndefined() } finally { aliveSpy.mockRestore() @@ -1494,6 +1518,173 @@ describe('PtyHandler', () => { expect(mockKill).toHaveBeenCalledWith('SIGKILL') expect(handler.activePtyCount).toBe(0) + }) + + // A relay that never knew an id may be a replacement whose predecessor's shells are still + // running, so "not found" proves nothing. A relay that WATCHED the shell exit knows first-hand. + // Keeping that observation is what lets the client tell a dead shell from an unknown one, and + // therefore what lets it respawn without risking a second agent on a live transcript. + // What the host answers reaches clients that predate the reply, and one of those reads an + // unrecognized attach error as neither death nor recovery — a stranded pane. So the proof is + // offered only to a client that says it can act on it. + /** `toThrow(expect.not.stringMatching(...))` passes whenever ANYTHING throws, so a negative + * claim about the wording has to read the message itself. */ + async function attachRejectionMessage(params: Record): Promise { + const error = await dispatcher.callRequest('pty.attach', params).then( + () => null, + (thrown: unknown) => thrown as Error + ) + expect(error, 'attach unexpectedly succeeded').toBeInstanceOf(Error) + return (error as Error).message + } + + it('keeps the older wording for a client that has not declared it understands exit proof', async () => { + const spawned = await spawnPty({}) + const onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as + | ((evt: { exitCode: number }) => void) + | undefined + onExitCb?.({ exitCode: 3 }) + + await expect(dispatcher.callRequest('pty.attach', { id: spawned.id })).rejects.toThrow( + `PTY "${'pty-1'}" not found` + ) + }) + + // The sharpest edge in the whole change. A caller that names no shell is not asking about this + // one, so a remembered exit is not an answer to it: relay A is killed leaving pty-1 alive and + // orphaned, relay B mints its own pty-1 and THAT one exits. A pane carrying no recorded identity + // must not be told its shell is gone, because the reply authorizes replacing a process that is + // still running — the double agent resume this whole change exists to prevent. + it('does not prove an exit to a caller that names no shell', async () => { + const spawned = await spawnPty({}) + const onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as + | ((evt: { exitCode: number }) => void) + | undefined + onExitCb?.({ exitCode: 3 }) + + expect(await attachRejectionMessage({ id: spawned.id, exitProofSupported: true })).not.toMatch( + /SSH_PTY_EXITED/ + ) + }) + + // The same rule on the other proof route. A replacement relay can hold a DIFFERENT shell under + // this id; if that one dies, its death is not evidence about the caller's, which may be running + // orphaned under the relay this one replaced. The reap still happens — a dead shell should be + // cleared whoever asked — but the answer must not authorize replacing a live process. + it('does not prove a reaped shell to a caller that named a different one', async () => { + const spawned = await spawnPty({}) + const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) + try { + const message = await attachRejectionMessage({ + id: spawned.id, + exitProofSupported: true, + expectedIncarnationId: 'a-shell-from-a-previous-relay' + }) + expect(message).not.toMatch(/SSH_PTY_EXITED/) + expect(message).toMatch(/identity mismatch/i) + } finally { + aliveSpy.mockRestore() + } + // The reap still ran: the dead entry is gone regardless of who asked. + expect(handler.activePtyCount).toBe(0) + }) + + it('answers a later attach with the exit it observed, not an unknown id', async () => { + const spawned = await spawnPty({}) + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + // The exit listener was captured at spawn by the shared mock; drive a real exit through it. + onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as typeof onExitCb + expect(onExitCb).toBeDefined() + onExitCb!({ exitCode: 3 }) + + const asking = { + id: spawned.id, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + } + await expect(attachPty(asking)).rejects.toThrow(/SSH_PTY_EXITED/) + await expect(attachPty(asking)).rejects.toThrow(/code=3/) + }) + + // The dangerous shape: a replaced relay hands pty-1 to a new shell, that one exits, and the + // pane still bound to the ORIGINAL pty-1 asks about its own. Reporting the stranger's death as + // proof would authorize a respawn onto a shell that may still be running. + it('does not report a remembered exit for a shell the caller did not ask about', async () => { + const spawned = await spawnPty({}) + const onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as + | ((evt: { exitCode: number }) => void) + | undefined + onExitCb?.({ exitCode: 3 }) + + expect( + await attachRejectionMessage({ + id: spawned.id, + exitProofSupported: true, + expectedIncarnationId: 'a-different-shell' + }) + ).not.toMatch(/SSH_PTY_EXITED/) + }) + + // INVERTED from the deleted `attachIdentityMismatches` suite. That guard compared the paneKey and + // tabId frozen at spawn; it caught a recycled id, but it also refused a pane that had merely moved + // to another tab — and refused as "not found", which the client read as death and answered by + // resuming the agent a second time. The property worth keeping is "a recycled id must not attach"; + // the property that had to go is "identity is where the pane lives". Both are pinned below, now + // against the shell's own incarnation. + it('refuses an attach whose expected incarnation names a different shell', async () => { + const first = await spawnPty({}) + + await expect( + dispatcher.callRequest('pty.attach', { + id: first.id, + expectedIncarnationId: 'incarnation-from-a-previous-relay' + }) + ).rejects.toThrow(/identity mismatch/i) + }) + + // The wording matters as much as the refusal: "not found" is what the client maps to an expired + // session, and expiry authorizes a respawn onto the very shell this branch proved is alive. + it('does not word an incarnation refusal as not-found', async () => { + const first = await spawnPty({}) + + expect( + await attachRejectionMessage({ + id: first.id, + expectedIncarnationId: 'incarnation-from-a-previous-relay' + }) + ).not.toMatch(/not found/i) + }) + + it('attaches when the expected incarnation is the shell it names', async () => { + const first = await spawnPty({}) + + await expect( + attachPty({ id: first.id, expectedIncarnationId: first.incarnationId }) + ).resolves.toMatchObject({ incarnationId: first.incarnationId }) + }) + + // An older client sends no expectation. Refusing there would strand every pane it owns. + it('stays permissive when the caller supplies no expected incarnation', async () => { + const first = await spawnPty({}) + + await expect(attachPty({ id: first.id })).resolves.toMatchObject({ + incarnationId: first.incarnationId + }) + }) + + // The regression the old tab-keyed guard caused. A pane carries its shell to a new tab; the + // relay still holds the paneKey/tabId frozen at spawn. That must not refuse the attach. + it('attaches a pane that has moved to another tab', async () => { + const first = await spawnPty({ paneKey: 'tab-a:0', tabId: 'tab-a' }) + + await expect( + attachPty({ + id: first.id, + expectedIncarnationId: first.incarnationId, + expectedPaneKey: 'tab-b:0', + expectedTabId: 'tab-b' + }) + ).resolves.toMatchObject({ incarnationId: first.incarnationId }) expect(vi.getTimerCount()).toBe(0) }) @@ -2202,13 +2393,17 @@ describe('PtyHandler', () => { expect(spawnOptions.env.ORCA_PANE_KEY).toBeUndefined() expect(spawnOptions.env.ORCA_TAB_ID).toBeUndefined() + // A caller that names no incarnation is an older client, and pane identity is the only fence + // it can be given — so it still applies, and still refuses. A caller that CAN name the shell is + // fenced on that instead and attaches regardless of which tab it now lives in; that is pinned + // by "attaches a pane that has moved to another tab". await expect( attachPty({ id: 'pty-1', expectedPaneKey: 'tab-b:leaf-b', expectedTabId: 'tab-b' }) - ).rejects.toThrow('PTY "pty-1" not found') + ).rejects.toThrow(/identity mismatch/i) await expect( attachPty({ @@ -3451,13 +3646,15 @@ describe('PtyHandler', () => { expect(callArgs.env.ORCA_PANE_KEY).toBeUndefined() expect(callArgs.env.ORCA_TAB_ID).toBeUndefined() + // Revive preserves the serialized pane identity, and for a caller that names no incarnation + // that identity is still the fence. await expect( dispatcher.callRequest('pty.attach', { id: 'pty-1', expectedPaneKey: 'tab-other:leaf', expectedTabId: 'tab-other' }) - ).rejects.toThrow('PTY "pty-1" not found') + ).rejects.toThrow(/identity mismatch/i) }) it('invokes the exit listener with the spawn-time paneKey', async () => { @@ -3769,13 +3966,19 @@ describe('PtyHandler', () => { mockPtySpawn.mockReturnValue({ ...mockPtyInstance, onData: vi.fn(), onExit: vi.fn() }) const poolEmpty = vi.fn() handler.onPtyPoolEmpty(poolEmpty) - await spawnPty() + const spawned = await spawnPty() const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) try { - await expect(attachPty({ id: 'pty-1', suppressReplayNotification: true })).rejects.toThrow( - 'PTY "pty-1" not found' - ) + // INVERTED with the reap above: proof of exit, not an unknown id. + await expect( + attachPty({ + id: 'pty-1', + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) } finally { aliveSpy.mockRestore() } @@ -3928,39 +4131,3 @@ describe('PtyHandler', () => { expect(handler.pendingPtyCreationCount).toBe(0) }) }) - -describe('attachIdentityMismatches', () => { - it('rejects a paneKey collision across relay generations', () => { - // Old lease expects tab-a's pane; the reset relay's pty-1 belongs to tab-b. - expect( - attachIdentityMismatches({ paneKey: 'tab-a:0' }, { paneKey: 'tab-b:0', tabId: 'tab-b' }) - ).toBe(true) - }) - - it('rejects a tabId collision when only tab identity is known', () => { - expect(attachIdentityMismatches({ tabId: 'tab-a' }, { tabId: 'tab-b' })).toBe(true) - }) - - it('accepts a matching identity', () => { - expect( - attachIdentityMismatches( - { paneKey: 'tab-a:0', tabId: 'tab-a' }, - { paneKey: 'tab-a:0', tabId: 'tab-a' } - ) - ).toBe(false) - }) - - it('stays permissive when the caller supplies no identity', () => { - expect(attachIdentityMismatches({}, { paneKey: 'tab-a:0', tabId: 'tab-a' })).toBe(false) - }) - - it('stays permissive when the managed PTY predates identity capture', () => { - expect(attachIdentityMismatches({ paneKey: 'tab-a:0', tabId: 'tab-a' }, {})).toBe(false) - }) - - it('does not reject on tabId when paneKey matches (split panes share a tab)', () => { - expect( - attachIdentityMismatches({ paneKey: 'tab-a:1' }, { paneKey: 'tab-a:1', tabId: 'tab-a' }) - ).toBe(false) - }) -}) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 3f8bd593e08..b63b1cb631c 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -26,6 +26,7 @@ import { normalizeRuntimePathForComparison } from '../shared/cross-platform-path' import { splitWorktreeId } from '../shared/worktree-id' +import { formatPtyExitedError } from '../shared/ssh-pty-failure-tokens' import { PhysicalExitTracker } from '../shared/physical-exit-tracker' import { SHELL_READY_MARKER_PREFIX } from '../main/shell-ready-marker-scanner' import { @@ -176,6 +177,12 @@ type RelayAgentSessionCreateResult = { sourceActivation?: PtySourceReceivingActivation } +/** Enough to cover every pane a host could plausibly have lost since the client last looked. */ +const MAX_REMEMBERED_PTY_EXITS = 256 + +/** No exit status to report: the relay found the pid gone rather than watching it go. */ +const PTY_EXIT_CODE_OBSERVED_GONE = -1 + const AGENT_SESSION_CREATE_OPERATION_ID_PATTERN = /^[A-Za-z0-9_-]{43}$/ const AGENT_SESSION_CREATE_OPERATION_RETENTION_MS = 24 * 60 * 60 * 1000 const AGENT_SESSION_CREATE_OPERATION_LIMIT = 4_096 @@ -342,12 +349,11 @@ export type PtyExitListener = (event: { id: string; paneKey?: string }) => void type PtyIdentity = { paneKey?: string; tabId?: string } -/** - * True when a reattach's expected pane identity contradicts the target PTY's own. - * Rejects cross-relay-generation id collisions (a reset relay reuses `pty-N`). - * Only compares fields present on both sides; absent identity stays permissive. - */ -export function attachIdentityMismatches(expected: PtyIdentity, managed: PtyIdentity): boolean { +/** Fallback fence for a client too old to name a shell by incarnation. It froze paneKey/tabId at + * spawn, so it refuses a pane that merely moved tabs — which is why a client that CAN name the + * shell is fenced on that instead. Not deleted, because the relay is shared: an upgraded host + * would otherwise leave every not-yet-updated client attaching a recycled id unchecked. */ +function attachIdentityMismatches(expected: PtyIdentity, managed: PtyIdentity): boolean { return Boolean( (expected.paneKey && managed.paneKey && expected.paneKey !== managed.paneKey) || (expected.tabId && managed.tabId && expected.tabId !== managed.tabId) @@ -377,6 +383,10 @@ export class PtyHandler { private outputFlushTimer: ReturnType | null = null private pendingOutputByPty = new Map() private pendingExitByPty = new Map() + /** Exits this process WATCHED, kept past publication so attach can prove death rather than report + * an unknown id. Bounded and oldest-evicted; losing one costs a user-asked respawn, never a + * wrong one, and a crash losing all of them correctly reads as no knowledge. */ + private exitedPtys = new Map() private pausedOutputPtys = new Set() private consumerPausedOutputPtys = new Set() private removeLegacyCapacityListener: (() => void) | null = null @@ -800,6 +810,7 @@ export class PtyHandler { code: exitCode, incarnationId: managed.incarnationId }) + this.rememberPtyExit(managed.id, exitCode, managed.incarnationId) this.publishPendingExit(managed.id) this.notifyExitListener(managed) this.agentSessionOwners.release(managed.id) @@ -810,6 +821,29 @@ export class PtyHandler { }) } + /** Proof must name the caller's own shell: present and matching. Both gone-paths ask this, so a + * change to the rule cannot reach one and miss the other. */ + private exitProofAnswersCaller( + expectedIncarnationId: string | undefined, + exitedIncarnationId: string, + clientUnderstandsExitProof: boolean + ): boolean { + return clientUnderstandsExitProof && expectedIncarnationId === exitedIncarnationId + } + + /** Oldest-evicted: a Map iterates in insertion order, so the first key is the oldest. */ + private rememberPtyExit(id: string, code: number, incarnationId: string): void { + this.exitedPtys.delete(id) + this.exitedPtys.set(id, { code, incarnationId }) + while (this.exitedPtys.size > MAX_REMEMBERED_PTY_EXITS) { + const oldest = this.exitedPtys.keys().next() + if (oldest.done) { + break + } + this.exitedPtys.delete(oldest.value) + } + } + private releaseRelayIngress(managed: ManagedPty): void { const startupCommand = managed.startupCommand if (startupCommand) { @@ -1677,9 +1711,29 @@ export class PtyHandler { sourceActivation?: PtySourceReceivingActivation }> { const id = params.id as string + const expectedIncarnationId = + typeof params.expectedIncarnationId === 'string' ? params.expectedIncarnationId : undefined + // Gated because the host's answer reaches clients predating it, which read an unrecognized + // attach error as neither death nor recovery and strand the pane. + const clientUnderstandsExitProof = params.exitProofSupported === true const managed = this.ptys.get(id) // Why: after dispose, pty.kill is a POSIX no-op; treat disposed as not-found so failures aren't silent. if (!managed || managed.disposed) { + // `disposed` is teardown, not an observed exit, so only a genuinely absent pty may be + // answered from what this process watched exit. + if (!managed) { + const exited = this.exitedPtys.get(id) + if ( + exited && + this.exitProofAnswersCaller( + expectedIncarnationId, + exited.incarnationId, + clientUnderstandsExitProof + ) + ) { + throw new Error(formatPtyExitedError(id, exited.code, exited.incarnationId)) + } + } throw new Error(`PTY "${id}" not found`) } @@ -1693,19 +1747,43 @@ export class PtyHandler { disposeManagedPty(managed) this.removePty(id) this.clearPtyFlowState(id) - throw new Error(`PTY "${id}" not found`) + // The reap runs whoever asked, but the ANSWER depends on whose shell died: under this id a + // replacement relay may hold a different one entirely, and the caller's may be orphaned and + // alive under the relay this one replaced. + this.rememberPtyExit(id, PTY_EXIT_CODE_OBSERVED_GONE, managed.incarnationId) + if (expectedIncarnationId && expectedIncarnationId !== managed.incarnationId) { + throw new Error(`PTY "${id}" identity mismatch`) + } + throw new Error( + this.exitProofAnswersCaller( + expectedIncarnationId, + managed.incarnationId, + clientUnderstandsExitProof + ) + ? formatPtyExitedError(id, PTY_EXIT_CODE_OBSERVED_GONE, managed.incarnationId) + : `PTY "${id}" not found` + ) } - // Why: generation resets can reuse PTY IDs; reject conflicting identities. - const mismatch = attachIdentityMismatches( - { - paneKey: typeof params.expectedPaneKey === 'string' ? params.expectedPaneKey : undefined, - tabId: typeof params.expectedTabId === 'string' ? params.expectedTabId : undefined - }, - managed.attachIdentity ?? { paneKey: managed.paneKey, tabId: managed.tabId } - ) - if (mismatch) { - throw new Error(`PTY "${id}" not found (identity mismatch)`) + // A reset relay restarts ids at pty-1, so an id alone can name somebody else's shell. The + // incarnation is the shell's own identity, so it catches that without caring where the pane + // lives — unlike the pane identity below, which froze the tab at spawn and refused moved panes. + if ( + !expectedIncarnationId && + attachIdentityMismatches( + { + paneKey: typeof params.expectedPaneKey === 'string' ? params.expectedPaneKey : undefined, + tabId: typeof params.expectedTabId === 'string' ? params.expectedTabId : undefined + }, + managed.attachIdentity ?? { paneKey: managed.paneKey, tabId: managed.tabId } + ) + ) { + throw new Error(`PTY "${id}" identity mismatch`) + } + if (expectedIncarnationId && expectedIncarnationId !== managed.incarnationId) { + // Deliberately NOT worded "not found": that phrasing is what the client maps to an expired + // session, and expiry authorizes a respawn onto a shell this branch just proved is alive. + throw new Error(`PTY "${id}" identity mismatch`) } managed.startupIngress?.snapshotBarrier() diff --git a/src/renderer/src/components/terminal-pane/TerminalPane.tsx b/src/renderer/src/components/terminal-pane/TerminalPane.tsx index 323a3dc497e..adcc8541f1a 100644 --- a/src/renderer/src/components/terminal-pane/TerminalPane.tsx +++ b/src/renderer/src/components/terminal-pane/TerminalPane.tsx @@ -190,7 +190,7 @@ import { scheduleImagePasteWebglAtlasRecovery } from './terminal-webgl-atlas-rec import { restoreTerminalFitToDesktop, restoreTerminalFitsToDesktop } from './terminal-fit-restore' import { useVisibleTerminalTabClaim } from './use-visible-terminal-tab-claim' import { TerminalSshReconnectOverlay } from './TerminalSshReconnectOverlay' -import { TerminalRemoteRuntimeReconnectBanner } from './TerminalRemoteRuntimeReconnectBanner' +import { TerminalPaneDisconnectedBanner } from './TerminalPaneDisconnectedBanner' import { selectTerminalTabAgentTypesByLeaf } from './terminal-tab-agent-type-index' import { canContinueAgentSessionInNewSession } from './terminal-agent-session-continuation' import { @@ -2915,6 +2915,10 @@ function TerminalPane( resolveAgentForLeaf(contextMenuLeafId) ) // Each toggle gates on its own leaf (header=active, menu=opened-over), so mixed splits show it only where chat can render. + // The pane-level disconnected card owns the bottom strip while it is up; see the error toast below. + const activePaneShowsUnreachableCard = Boolean( + activePane && ptyRecoveryStatesByPaneId[activePane.id]?.unreachablePane + ) const activePaneCanToggleChat = canToggleChatForLeaf(activePane?.leafId ?? null) const contextMenuCanToggleChat = canToggleChatForLeaf(contextMenuLeafId) return ( @@ -2982,8 +2986,12 @@ function TerminalPane( ) })} {/* Why: the reconnect banner already owns SSH recovery UX; the z-50 error - toast was painting over it (same bottom strip) with the raw ssh:connect failure. */} - {terminalError && isActive && !showSshReconnectOverlay ? ( + toast was painting over it (same bottom strip) with the raw ssh:connect failure. + The pane's own disconnected card owns the same strip and the same story, and the toast + outranks it at z-50 — so it covered BOTH of its actions and made the affordance + unclickable. Naming only the connection overlay here was the gap: the card is raised for + a per-pane failure that leaves the connection up, which is exactly when this fires. */} + {terminalError && isActive && !showSshReconnectOverlay && !activePaneShowsUnreachableCard ? ( { + const unreachable = recoveryState.unreachablePane + if (unreachable) { + unreachable.onRetry() + return + } paneTransportsRef.current.get(pane.id)?.retryRecovery?.() }} + onStartNewTerminal={recoveryState.unreachablePane?.onStartNewTerminal} + onRestoreTerminalFocus={() => pane.terminal.focus()} />, pane.container, `remote-runtime-reconnect-${pane.id}` diff --git a/src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx b/src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx new file mode 100644 index 00000000000..a8229c9594e --- /dev/null +++ b/src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx @@ -0,0 +1,164 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' +import { cleanup, render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { TooltipProvider } from '@/components/ui/tooltip' +import { TerminalPaneDisconnectedBanner } from './TerminalPaneDisconnectedBanner' + +vi.mock('@/i18n/i18n', () => ({ + translate: (_key: string, fallback: string) => fallback +})) + +afterEach(cleanup) + +function renderBanner(banner: React.ReactNode): ReturnType { + return render({banner}) +} + +describe('TerminalPaneDisconnectedBanner', () => { + it('shows quiet bounded automatic recovery without a manual action', () => { + renderBanner( + + ) + + expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument() + expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument() + expect(screen.queryByRole('button')).not.toBeInTheDocument() + }) + + it('offers one explicit reconnect action after automatic recovery stops', async () => { + const onReconnect = vi.fn() + const user = userEvent.setup() + renderBanner( + + ) + + expect(screen.getByText('Remote runtime disconnected')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Reconnect' })) + expect(onReconnect).toHaveBeenCalledOnce() + }) +}) + +/** + * STA-3077 step E-0. A reattach that fails without proving the shell is gone used to either + * respawn silently — resuming the same agent a second time into one transcript — or leave the pane + * frozen behind a raw error toast. It now renders here, with the two actions the design approved. + */ +describe('TerminalPaneDisconnectedBanner, unreachable SSH pane', () => { + function renderUnreachable() { + const onReconnect = vi.fn() + const onStartNewTerminal = vi.fn() + const onRestoreTerminalFocus = vi.fn() + renderBanner( + + ) + return { onReconnect, onStartNewTerminal, onRestoreTerminalFocus } + } + + it('offers exactly the two approved actions', async () => { + const user = userEvent.setup() + const { onReconnect, onStartNewTerminal } = renderUnreachable() + + const buttons = screen.getAllByRole('button') + expect(buttons.map((button) => button.textContent)).toEqual([ + 'Try again', + 'Start a new terminal' + ]) + + await user.click(screen.getByRole('button', { name: 'Try again' })) + expect(onReconnect).toHaveBeenCalledOnce() + await user.click(screen.getByRole('button', { name: 'Start a new terminal' })) + expect(onStartNewTerminal).toHaveBeenCalledOnce() + }) + + it('returns keyboard focus to this terminal after either action', async () => { + const user = userEvent.setup() + const onReconnect = vi.fn() + const onStartNewTerminal = vi.fn() + renderBanner( + <> +