From ede69ffc7f9963cc0350c3a282df26622c64ef04 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:21:06 -0700 Subject: [PATCH 01/87] perf(skills): bound and share skill discovery scans (#14204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skill discovery re-walked every skill root on every window focus, pane mount, and connected client. The root set was already bounded; what was not bounded was how often and how redundantly it was walked. - Focus called refresh(true), bypassing every cache down to a disk walk. - The process that owns the disk had no cache and no in-flight dedup. - Panes with different cwds each re-walked the same 12 home roots. - Fan-out inside a scan was unbounded, and every package was walked twice (once to find SKILL.md, once to count its files, node_modules included). Adds one coalescing primitive — in-flight dedup plus a short TTL behind a bounded LRU — used for per-target dedup below both the IPC and RPC entry points, per-root sharing on the native path, and whole-result reuse on the WSL path. A scan may publish only while it still owns its pending slot, so a scan begun before an invalidation can never re-cache a pre-mutation result. Bounds per-skill fan-out to the existing candidate concurrency limit, and bounds the package file walk by depth with a node_modules prune. Focus now reads through a 15s freshness window; explicit signals (install completed, Settings Refresh, native-chat Retry, terminal exit) set a new optional `refresh` wire field that bypasses every cache, including on remote runtimes. Measured on a 32-concurrent-scan burst across 8 workspaces: 134,880 -> 2,956 filesystem calls and 1080ms -> 45ms, same 31 skills returned. --- src/main/ipc/skills.test.ts | 11 +- src/main/ipc/skills.ts | 11 +- src/main/runtime/rpc/methods/skills.test.ts | 21 +- src/main/runtime/rpc/methods/skills.ts | 4 +- src/main/skills/discovery.test.ts | 10 +- src/main/skills/discovery.ts | 290 +++++++----------- .../skill-discovery-concurrency.test.ts | 195 ++++++++++++ .../skills/skill-discovery-target.test.ts | 158 ++++++++++ src/main/skills/skill-discovery-target.ts | 73 ++++- src/main/skills/skill-root-file-walk.test.ts | 74 +++++ src/main/skills/skill-root-file-walk.ts | 145 +++++++++ src/main/skills/skill-scan-coalescer.test.ts | 222 ++++++++++++++ src/main/skills/skill-scan-coalescer.ts | 121 ++++++++ .../use-native-chat-skills.react.test.tsx | 27 +- .../native-chat/use-native-chat-skills.ts | 21 +- .../src/components/skills/SkillsPage.tsx | 66 ++-- ...talled-agent-skill-discovery-cache.test.ts | 28 +- .../installed-agent-skill-discovery-cache.ts | 40 ++- .../hooks/installed-agent-skill-discovery.ts | 5 +- .../useInstalledAgentSkills.react.test.tsx | 73 +++-- .../src/hooks/useInstalledAgentSkills.ts | 25 +- .../src/runtime/runtime-skills-client.test.ts | 20 ++ .../src/runtime/runtime-skills-client.ts | 8 +- src/shared/skills.ts | 6 +- 24 files changed, 1386 insertions(+), 268 deletions(-) create mode 100644 src/main/skills/skill-discovery-concurrency.test.ts create mode 100644 src/main/skills/skill-discovery-target.test.ts create mode 100644 src/main/skills/skill-root-file-walk.test.ts create mode 100644 src/main/skills/skill-root-file-walk.ts create mode 100644 src/main/skills/skill-scan-coalescer.test.ts create mode 100644 src/main/skills/skill-scan-coalescer.ts diff --git a/src/main/ipc/skills.test.ts b/src/main/ipc/skills.test.ts index e6f88345ff2..b85abfe276d 100644 --- a/src/main/ipc/skills.test.ts +++ b/src/main/ipc/skills.test.ts @@ -28,7 +28,8 @@ vi.mock('electron', () => ({ })) vi.mock('../skills/discovery', () => ({ - discoverSkills: discoverSkillsMock + discoverSkills: discoverSkillsMock, + clearSkillRootScanCache: vi.fn() })) vi.mock('../skills/skill-discovery-wsl', () => ({ @@ -131,7 +132,7 @@ describe('registerSkillsHandlers', () => { } }) - expect(discoverSkillsMock).toHaveBeenCalledWith({ repos }) + expect(discoverSkillsMock).toHaveBeenCalledWith({ repos, refresh: false }) expect(getWslHomeMock).not.toHaveBeenCalled() }) @@ -140,7 +141,11 @@ describe('registerSkillsHandlers', () => { await handler(null, { cwd: '/repo/worktree' }) - expect(discoverSkillsMock).toHaveBeenCalledWith({ repos: [], cwd: '/repo/worktree' }) + expect(discoverSkillsMock).toHaveBeenCalledWith({ + repos: [], + cwd: '/repo/worktree', + refresh: false + }) }) it('uses the selected project WSL distro for skill discovery', async () => { diff --git a/src/main/ipc/skills.ts b/src/main/ipc/skills.ts index e2d12ca57ba..746f02416d1 100644 --- a/src/main/ipc/skills.ts +++ b/src/main/ipc/skills.ts @@ -15,6 +15,7 @@ import { SkillUpdateRunner } from '../skills/skill-update-run' import { skillUpdateFailedNames } from '../skills/skill-update-outcome' import { readGloballyUpdatableSkillLocks } from '../skills/skill-update-registration' import { + clearSkillDiscoveryCaches, discoverSkillsOnTarget, resolveSkillDiscoveryTarget } from '../skills/skill-discovery-target' @@ -32,6 +33,9 @@ export function registerSkillsHandlers(store: Store): void { // Why: per-skill outcomes come from re-hashing what is actually on disk, not // from scraping stdout. rescanOutdatedNames: async (names) => { + // Why: the run just rewrote skill packages on this host. Clients that never + // send `refresh` (older builds) would otherwise read a pre-run scan. + clearSkillDiscoveryCaches() // The lock read is fresh on purpose: the run just rewrote it, and the // verdict accepts unrecognized content only when disk matches that record. const [inventory, globalSkillLocks] = await Promise.all([ @@ -53,7 +57,12 @@ export function registerSkillsHandlers(store: Store): void { 'skills:discover', async (_event, target?: SkillDiscoveryTarget): Promise => { const parsedTarget = target ? SkillDiscoveryTargetSchema.parse(target) : undefined - return discoverSkillsOnTarget(resolveSkillDiscoveryTarget(parsedTarget), store.getRepos()) + return discoverSkillsOnTarget( + resolveSkillDiscoveryTarget(parsedTarget), + store.getRepos(), + // Why: only a caller that knows disk changed may bypass the shared scans. + { refresh: parsedTarget?.refresh === true } + ) } ) diff --git a/src/main/runtime/rpc/methods/skills.test.ts b/src/main/runtime/rpc/methods/skills.test.ts index a4335ec2e62..cd82be0ca3a 100644 --- a/src/main/runtime/rpc/methods/skills.test.ts +++ b/src/main/runtime/rpc/methods/skills.test.ts @@ -7,7 +7,10 @@ vi.mock('../../../skills/skill-discovery-target', () => ({ })) import { SKILL_METHODS } from './skills' -import { resolveSkillDiscoveryTarget } from '../../../skills/skill-discovery-target' +import { + discoverSkillsOnTarget, + resolveSkillDiscoveryTarget +} from '../../../skills/skill-discovery-target' const WSL_RUNTIME = { status: 'resolved', @@ -65,4 +68,20 @@ describe('skills.discover RPC', () => { expect.objectContaining({ projectRuntime: WSL_RUNTIME }) ) }) + + it('only bypasses the host scan cache when the caller asks for a refresh', async () => { + await discoverMethod().handler({ cwd: '/repo' }, makeContext({})) + expect(vi.mocked(discoverSkillsOnTarget)).toHaveBeenLastCalledWith(expect.anything(), [], { + refresh: false + }) + + await discoverMethod().handler({ cwd: '/repo', refresh: true }, makeContext({})) + expect(vi.mocked(discoverSkillsOnTarget)).toHaveBeenLastCalledWith(expect.anything(), [], { + refresh: true + }) + }) + + it('accepts a params payload from an older client that cannot send refresh', () => { + expect(discoverMethod().params?.parse({ cwd: '/repo' })).toEqual({ cwd: '/repo' }) + }) }) diff --git a/src/main/runtime/rpc/methods/skills.ts b/src/main/runtime/rpc/methods/skills.ts index 346198aeffc..f6e5acbe6d9 100644 --- a/src/main/runtime/rpc/methods/skills.ts +++ b/src/main/runtime/rpc/methods/skills.ts @@ -19,7 +19,9 @@ export const SKILL_METHODS: RpcMethod[] = [ ...params, projectRuntime: runtime.resolveProjectRuntimeForWorktree(params.worktreeId) } - return discoverSkillsOnTarget(resolveSkillDiscoveryTarget(target), runtime.listRepos()) + return discoverSkillsOnTarget(resolveSkillDiscoveryTarget(target), runtime.listRepos(), { + refresh: params.refresh === true + }) } }) ] diff --git a/src/main/skills/discovery.test.ts b/src/main/skills/discovery.test.ts index 9f94e9f8347..55dcb55ed36 100644 --- a/src/main/skills/discovery.test.ts +++ b/src/main/skills/discovery.test.ts @@ -1,8 +1,8 @@ import { mkdtemp, mkdir, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { describe, expect, it } from 'vitest' -import { buildSkillDiscoverySources, discoverSkills } from './discovery' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { buildSkillDiscoverySources, clearSkillRootScanCache, discoverSkills } from './discovery' import { TUI_AGENT_CONFIG } from '../../shared/tui-agent-config' import type { Repo } from '../../shared/types' @@ -18,6 +18,12 @@ function makeRepo(path: string, connectionId: string | null = null): Repo { } } +beforeEach(() => { + // Roots are shared between scans for a few seconds; each case owns its own tree. + clearSkillRootScanCache() + vi.spyOn(console, 'info').mockImplementation(() => undefined) +}) + describe('skill discovery', () => { it('discovers home and repo SKILL.md packages with provider metadata', async () => { const root = await mkdtemp(join(tmpdir(), 'orca-skills-')) diff --git a/src/main/skills/discovery.ts b/src/main/skills/discovery.ts index 00edfc8e705..07bebe52b29 100644 --- a/src/main/skills/discovery.ts +++ b/src/main/skills/discovery.ts @@ -1,7 +1,6 @@ -import type { Dirent } from 'node:fs' -import { open, readdir, realpath, stat } from 'node:fs/promises' +import { open, realpath, stat } from 'node:fs/promises' import { homedir } from 'node:os' -import { basename, dirname, isAbsolute, join, relative, sep } from 'node:path' +import { basename, dirname, relative, sep } from 'node:path' import { summarizeSkillMarkdown } from '../../shared/skill-metadata' import type { Repo } from '../../shared/types' import type { @@ -18,12 +17,41 @@ import { type SkillScanRoot } from './skill-discovery-sources' import { discoverClaudePluginSkillSources } from './claude-plugin-skill-sources' +import { countPackageFiles, findSkillFiles } from './skill-root-file-walk' +import { runSkillCandidateTasks } from './skill-candidate-concurrency' +import { SkillScanCoalescer, type SkillScanOutcome } from './skill-scan-coalescer' export { buildSkillDiscoverySources } from './skill-discovery-sources' -const SKILL_FILE_NAME = 'SKILL.md' const MAX_MARKDOWN_BYTES = 256 * 1024 -const MAX_SKILL_FILES = 200 +// Why: the fixed home roots are identical for every target, so one worktree pane +// per open workspace used to re-walk the same directories once per pane. Sharing +// them for a few seconds is what bounds that fan-out. +export const SKILL_ROOT_SCAN_TTL_MS = 10_000 +// Why: sized off the root formula, not a round number. One scan builds +// `12 fixed home roots + 2 per local repo (+ cwd) + plugin roots`, so a bound +// smaller than a single scan's root count makes that scan evict its own earlier +// entries and the cache degrades to a ~0% hit rate — exactly the walk this +// exists to prevent. The live key space is the union across targets — the fixed +// home roots plus two per repo plus two per distinct workspace cwd — so this holds +// a few hundred repos with panes open, not an unbounded install. Past that the LRU +// keeps the hot home roots and the repo roots thrash, which degrades rather than +// breaks. Most repo roots do not exist, and a missing root caches as +// `{exists: false, skills: []}`. +const MAX_CACHED_SKILL_ROOTS = 1_024 +// Why: roots grow with the repo count, so an uncapped id list would make one log +// line grow with the install. Root *ids* are safe to log where labels and paths +// are not — a repo/plugin id is already a hash. +export const MAX_LOGGED_ROOT_IDS = 12 + +type RootScan = { exists: boolean; skills: ScannedSkill[] } + +const rootScans = new SkillScanCoalescer(MAX_CACHED_SKILL_ROOTS) + +/** Drop every shared root scan, e.g. after a skill install/update mutates disk. */ +export function clearSkillRootScanCache(): void { + rootScans.clear() +} async function pathExists(pathValue: string): Promise { try { @@ -34,129 +62,6 @@ async function pathExists(pathValue: string): Promise { } } -function isWithinDepth(rootPath: string, childPath: string, maxDepth: number): boolean { - const rel = relative(rootPath, childPath) - if (!rel) { - return true - } - // Why: `..cache` is a valid child name; only a real parent traversal escapes. - if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { - return false - } - return rel.split(sep).length <= maxDepth -} - -async function findSkillFiles(rootPath: string, maxDepth: number): Promise { - const out: string[] = [] - const visitedDirectoryPaths = new Set() - async function visit(dirPath: string): Promise { - if (!isWithinDepth(rootPath, dirPath, maxDepth)) { - return - } - let resolvedDirPath: string - try { - resolvedDirPath = await realpath(dirPath) - } catch { - return - } - if (visitedDirectoryPaths.has(resolvedDirPath)) { - return - } - visitedDirectoryPaths.add(resolvedDirPath) - - let entries: Dirent[] - try { - entries = await readdir(dirPath, { withFileTypes: true }) - } catch { - return - } - for (const entry of entries) { - const entryPath = join(dirPath, entry.name) - if (entry.name === SKILL_FILE_NAME) { - if (entry.isFile()) { - out.push(entryPath) - continue - } - if (entry.isSymbolicLink()) { - try { - if ((await stat(entryPath)).isFile()) { - out.push(entryPath) - } - } catch { - // Broken links are not valid skill files. - } - } - continue - } - if (entry.isDirectory()) { - await visit(entryPath) - continue - } - if (entry.isSymbolicLink()) { - // Why: users commonly symlink agent skill dirs across providers; follow - // directory links but guard by realpath so recursive links cannot loop. - try { - if ((await stat(entryPath)).isDirectory()) { - await visit(entryPath) - } - } catch { - // Broken links are not valid skill directories. - } - } - } - } - await visit(rootPath) - return out -} - -async function countFiles(dirPath: string): Promise { - let count = 0 - const visitedDirectoryPaths = new Set() - async function visit(currentPath: string): Promise { - if (count >= MAX_SKILL_FILES) { - return - } - let resolvedPath: string - try { - resolvedPath = await realpath(currentPath) - } catch { - return - } - if (visitedDirectoryPaths.has(resolvedPath)) { - return - } - visitedDirectoryPaths.add(resolvedPath) - - let entries: Dirent[] - try { - entries = await readdir(currentPath, { withFileTypes: true }) - } catch { - return - } - for (const entry of entries) { - if (count >= MAX_SKILL_FILES) { - return - } - const entryPath = join(currentPath, entry.name) - if (entry.isFile()) { - count += 1 - } else if (entry.isDirectory()) { - await visit(entryPath) - } else if (entry.isSymbolicLink()) { - try { - if ((await stat(entryPath)).isFile()) { - count += 1 - } - } catch { - // Broken links do not contribute to the skill package file count. - } - } - } - } - await visit(dirPath) - return count -} - async function readSkillSummary(skillFilePath: string): Promise<{ name: string | null description: string | null @@ -187,8 +92,11 @@ type ScannedSkill = DiscoveredSkill & { canonicalSkillFilePath: string } async function scanRoot(root: SkillScanRoot): Promise { const maxDepth = root.sourceKind === 'plugin' ? 9 : 4 const skillFiles = await findSkillFiles(root.path, maxDepth) - const skills = await Promise.all( - skillFiles.map(async (skillFilePath): Promise => { + // Why: a root can hold many packages and each one costs a summary read plus a + // package walk. Unbounded fan-out here is what turned one scan into a burst of + // filesystem-metadata work across every core. + const skills = await runSkillCandidateTasks( + skillFiles.map((skillFilePath) => async (): Promise => { // Why: path identity belongs to the scanning host; canonicalizing before // returning prevents symlinked roots from becoming duplicate picker rows. const canonicalSkillFilePath = await realpath(skillFilePath).catch(() => skillFilePath) @@ -211,7 +119,7 @@ async function scanRoot(root: SkillScanRoot): Promise { directoryPath, skillFilePath, installed: true, - fileCount: await countFiles(directoryPath), + fileCount: await countPackageFiles(directoryPath), updatedAt: summary.updatedAt, canonicalSkillFilePath } satisfies ScannedSkill @@ -220,13 +128,66 @@ async function scanRoot(root: SkillScanRoot): Promise { return skills.filter((skill): skill is ScannedSkill => skill !== null) } +// Why: two roots can share a path (e.g. `~/.claude/skills` is both a home root +// and a repo root when the home dir is the workspace), and their scan differs +// only by depth, which `sourceKind` decides. +function rootScanKey(root: SkillScanRoot): string { + return `${root.sourceKind}\0${root.path}` +} + +function scanRootShared( + root: SkillScanRoot, + refresh: boolean +): Promise> { + return rootScans.run(rootScanKey(root), { ttlMs: SKILL_ROOT_SCAN_TTL_MS, refresh }, async () => { + const exists = await pathExists(root.path) + return { exists, skills: exists ? await scanRoot(root) : [] } + }) +} + +function mergeScannedSkill(seen: Map, skill: ScannedSkill): void { + // Why: overlapping repo/cwd roots and symlinked provider homes can reach + // the same file. Keep the first source's higher-level scope identity, but + // record every contributing root so per-agent visibility survives dedup. + const existing = seen.get(skill.canonicalSkillFilePath) + if (!existing) { + const { canonicalSkillFilePath, ...publicSkill } = skill + // Copy: a shared root scan hands the same skill object to every caller, so the + // result each one owns must not alias the cached arrays. + seen.set(canonicalSkillFilePath, { + ...publicSkill, + providers: [...publicSkill.providers], + rootPaths: [skill.rootPath] + }) + return + } + if (existing.rootPaths && !existing.rootPaths.includes(skill.rootPath)) { + existing.rootPaths.push(skill.rootPath) + } + // Why: providers is per-agent visibility just like rootPaths; keeping only + // the first root's tags makes a shared/symlinked skill under-report which + // agents can see it on the Settings provider badges/filter. Reassign a + // fresh array — `providers` aliases the scan root's array, so pushing in + // place would mutate the root and every sibling skill/source sharing it. + const mergedProviders = [...existing.providers] + for (const provider of skill.providers) { + if (!mergedProviders.includes(provider)) { + mergedProviders.push(provider) + } + } + existing.providers = mergedProviders +} + export async function discoverSkills(args: { repos?: Repo[] homeDir?: string cwd?: string includeCwd?: boolean + refresh?: boolean }): Promise { + const startedAt = Date.now() const homeDir = args.homeDir ?? homedir() + const refresh = args.refresh === true const roots = [ ...buildSkillDiscoverySources({ ...args, homeDir }), // Why: plugin discovery is native-chat data keyed to an explicit workspace. @@ -235,51 +196,36 @@ export async function discoverSkills(args: { ? await discoverClaudePluginSkillSources({ homeDir, cwd: args.cwd }) : []) ] - const sources: SkillDiscoverySource[] = [] - const skillGroups = await Promise.all( - roots.map(async (root) => { - const exists = await pathExists(root.path) - sources.push({ - ...root, - providers: [...root.providers], - exists, - skippedReason: exists ? undefined : 'missing' - }) - if (!exists) { - return [] - } - return scanRoot(root) - }) - ) + const scans = await Promise.all(roots.map((root) => scanRootShared(root, refresh))) + const sources: SkillDiscoverySource[] = roots.map((root, index) => ({ + ...root, + providers: [...root.providers], + exists: scans[index].value.exists, + skippedReason: scans[index].value.exists ? undefined : 'missing' + })) const seen = new Map() - for (const skill of skillGroups.flat()) { - // Why: overlapping repo/cwd roots and symlinked provider homes can reach - // the same file. Keep the first source's higher-level scope identity, but - // record every contributing root so per-agent visibility survives dedup. - const existing = seen.get(skill.canonicalSkillFilePath) - if (existing) { - if (existing.rootPaths && !existing.rootPaths.includes(skill.rootPath)) { - existing.rootPaths.push(skill.rootPath) - } - // Why: providers is per-agent visibility just like rootPaths; keeping only - // the first root's tags makes a shared/symlinked skill under-report which - // agents can see it on the Settings provider badges/filter. Reassign a - // fresh array — `providers` aliases the scan root's array, so pushing in - // place would mutate the root and every sibling skill/source sharing it. - const mergedProviders = [...existing.providers] - for (const provider of skill.providers) { - if (!mergedProviders.includes(provider)) { - mergedProviders.push(provider) - } - } - existing.providers = mergedProviders - continue + for (const { value } of scans) { + for (const skill of value.skills) { + mergeScannedSkill(seen, skill) } - const { canonicalSkillFilePath, ...publicSkill } = skill - seen.set(canonicalSkillFilePath, { ...publicSkill, rootPaths: [skill.rootPath] }) + } + const skills = Array.from(seen.values()).sort(compareSkills) + // Why: root *ids* — a repo/plugin id is already a hash, while its label carries + // the repo or plugin name and its path carries the user's directory names. A + // fully cached scan did no filesystem work, so it stays silent rather than + // burying the bursts this line exists to make visible. + const walked = roots.filter((_, index) => !scans[index].cached).map((root) => root.id) + if (walked.length > 0) { + // `present` is not derivable from the rest: "walked 500 roots, 3 existed" is + // the shape that says the root set, not the tree, is what costs. The id list + // is capped because roots grow with the repo count. + const present = sources.filter((source) => source.exists).length + console.info( + `[skills] scan roots=${roots.length} present=${present} walked=${walked.length} skills=${skills.length} ms=${Date.now() - startedAt} ids=${walked.slice(0, MAX_LOGGED_ROOT_IDS).join(',')}` + ) } return { - skills: Array.from(seen.values()).sort(compareSkills), + skills, sources: sources.sort((a, b) => a.label.localeCompare(b.label, undefined, { sensitivity: 'base' }) ), diff --git a/src/main/skills/skill-discovery-concurrency.test.ts b/src/main/skills/skill-discovery-concurrency.test.ts new file mode 100644 index 00000000000..162ea10cf89 --- /dev/null +++ b/src/main/skills/skill-discovery-concurrency.test.ts @@ -0,0 +1,195 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, sep } from 'node:path' +import type * as FsPromises from 'node:fs/promises' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +// Why: the regression is measured in filesystem syscalls, not in returned data, so +// the walk's own readdir is what the assertions below count. +const { readdirPaths } = vi.hoisted(() => ({ readdirPaths: [] as string[] })) + +vi.mock('node:fs/promises', async () => { + const actual = await vi.importActual('node:fs/promises') + return { + ...actual, + readdir: (path: Parameters[0], ...rest: unknown[]) => { + readdirPaths.push(String(path)) + return (actual.readdir as (...args: unknown[]) => unknown)(path, ...rest) + } + } +}) + +const { clearSkillRootScanCache, discoverSkills, MAX_LOGGED_ROOT_IDS } = await import('./discovery') + +async function writeSkill(directory: string, name: string): Promise { + await mkdir(directory, { recursive: true }) + await writeFile(join(directory, 'SKILL.md'), `---\nname: ${name}\ndescription: d\n---\n`) +} + +/** A home with three provider roots populated, plus `paneCount` workspace roots. */ +async function buildFixture( + paneCount: number +): Promise<{ home: string; panes: string[]; noWorkspace: string }> { + const root = await mkdtemp(join(tmpdir(), 'orca-skill-concurrency-')) + const home = join(root, 'home') + await writeSkill(join(home, '.agents', 'skills', 'shared'), 'shared') + await writeSkill(join(home, '.claude', 'skills', 'review'), 'review') + await writeSkill(join(home, '.codex', 'skills', 'plan'), 'plan') + const panes: string[] = [] + for (let index = 0; index < paneCount; index += 1) { + const pane = join(root, `pane-${index}`) + await writeSkill(join(pane, '.agents', 'skills', `pane-${index}`), `pane-${index}`) + panes.push(pane) + } + // Why: with no cwd the source builder falls back to process.cwd(), which would + // drag this repo's own skills into every count below. + return { home, panes, noWorkspace: join(root, 'no-workspace') } +} + +// One populated root costs three readdirs per real walk: the root itself, the +// package directory while looking for SKILL.md, and the package directory again +// while counting its files. Anything above that is a root walked more than once. +const READDIR_CALLS_PER_POPULATED_ROOT = 3 + +function readdirCountUnder(path: string): number { + return readdirPaths.filter( + (candidate) => candidate === path || candidate.startsWith(`${path}${sep}`) + ).length +} + +beforeEach(() => { + clearSkillRootScanCache() + readdirPaths.length = 0 + vi.spyOn(console, 'info').mockImplementation(() => undefined) +}) + +afterEach(() => { + clearSkillRootScanCache() + vi.restoreAllMocks() +}) + +describe('bounded concurrent skill discovery', () => { + it('collapses a burst of identical scans into one walk of each root', async () => { + const { home, noWorkspace } = await buildFixture(0) + const claudeRoot = join(home, '.claude', 'skills') + + const results = await Promise.all( + Array.from({ length: 32 }, () => + discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + ) + ) + + expect(results).toHaveLength(32) + for (const result of results) { + expect(result.skills.map((skill) => skill.name).sort()).toEqual(['plan', 'review', 'shared']) + } + expect(readdirCountUnder(claudeRoot)).toBe(READDIR_CALLS_PER_POPULATED_ROOT) + }) + + it('shares the fixed home roots across panes that only differ by workspace', async () => { + const { home, panes } = await buildFixture(8) + + const results = await Promise.all( + panes.flatMap((pane) => + Array.from({ length: 4 }, () => discoverSkills({ homeDir: home, repos: [], cwd: pane })) + ) + ) + + expect(results).toHaveLength(32) + // Every pane still sees the shared home skills plus its own workspace skill. + for (const [index, result] of results.entries()) { + const paneIndex = Math.floor(index / 4) + expect(result.skills.map((skill) => skill.name).sort()).toEqual([ + `pane-${paneIndex}`, + 'plan', + 'review', + 'shared' + ]) + } + // The home roots are walked once for all 32 scans; only the per-pane roots repeat. + expect(readdirCountUnder(join(home, '.agents', 'skills'))).toBe( + READDIR_CALLS_PER_POPULATED_ROOT + ) + expect(readdirCountUnder(join(home, '.claude', 'skills'))).toBe( + READDIR_CALLS_PER_POPULATED_ROOT + ) + expect(readdirCountUnder(join(home, '.codex', 'skills'))).toBe(READDIR_CALLS_PER_POPULATED_ROOT) + for (const pane of panes) { + expect(readdirCountUnder(join(pane, '.agents', 'skills'))).toBe( + READDIR_CALLS_PER_POPULATED_ROOT + ) + } + }) + + it('reports missing roots without walking them', async () => { + const { home, noWorkspace } = await buildFixture(0) + + const result = await discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + + const missing = result.sources.find((source) => source.id === 'home-cursor') + expect(missing?.exists).toBe(false) + expect(missing?.skippedReason).toBe('missing') + expect(readdirCountUnder(join(home, '.cursor', 'skills'))).toBe(0) + }) + + it('re-reads disk when a caller refreshes, and serves the new result afterwards', async () => { + const { home, noWorkspace } = await buildFixture(0) + await discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + await writeSkill(join(home, '.agents', 'skills', 'added-later'), 'added-later') + + const stale = await discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + expect(stale.skills.map((skill) => skill.name)).not.toContain('added-later') + + const refreshed = await discoverSkills({ + homeDir: home, + repos: [], + cwd: noWorkspace, + refresh: true + }) + expect(refreshed.skills.map((skill) => skill.name)).toContain('added-later') + + // Why: asserting only that the skill is present would pass with no caching at + // all — the roots would simply re-walk and find it. The refresh has to leave + // every root cached, or the next scan re-walks the whole set. + readdirPaths.length = 0 + const afterRefresh = await discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + expect(afterRefresh.skills.map((skill) => skill.name)).toContain('added-later') + expect(readdirPaths).toEqual([]) + }) + + it('gives every caller its own arrays so a shared scan cannot be mutated across results', async () => { + const { home, noWorkspace } = await buildFixture(0) + + const [first, second] = await Promise.all([ + discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }), + discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + ]) + + const firstShared = first.skills.find((skill) => skill.name === 'shared') + const secondShared = second.skills.find((skill) => skill.name === 'shared') + expect(firstShared?.providers).not.toBe(secondShared?.providers) + expect(firstShared?.rootPaths).not.toBe(secondShared?.rootPaths) + }) + + it('logs the roots it walked, by id, and never a filesystem path', async () => { + const { home, noWorkspace } = await buildFixture(0) + const info = vi.spyOn(console, 'info').mockImplementation(() => undefined) + + await discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + + const line = String(info.mock.calls.at(0)?.at(0)) + // `present` is the signal that separates "big tree" from "big root set", and + // is not derivable from the other counts. + expect(line).toContain('[skills] scan roots=14 present=3 walked=14 skills=3') + expect(line).toContain('home-claude') + expect(line).not.toContain(home) + expect(line).not.toContain(tmpdir()) + // The id list is capped so one line cannot grow with the repo count. + expect(line.slice(line.indexOf('ids=')).split(',')).toHaveLength(MAX_LOGGED_ROOT_IDS) + + info.mockClear() + await discoverSkills({ homeDir: home, repos: [], cwd: noWorkspace }) + // A fully cached scan did no filesystem work, so it must stay silent. + expect(info).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/skills/skill-discovery-target.test.ts b/src/main/skills/skill-discovery-target.test.ts new file mode 100644 index 00000000000..f136536b189 --- /dev/null +++ b/src/main/skills/skill-discovery-target.test.ts @@ -0,0 +1,158 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SkillDiscoveryResult } from '../../shared/skills' +import type { Repo } from '../../shared/types' + +const { nativeScans, wslScans } = vi.hoisted(() => ({ + nativeScans: [] as unknown[], + wslScans: [] as unknown[] +})) + +const emptyResult = (): SkillDiscoveryResult => ({ skills: [], sources: [], scannedAt: 1 }) + +vi.mock('./discovery', () => ({ + clearSkillRootScanCache: vi.fn(), + discoverSkills: vi.fn(async (args: unknown) => { + nativeScans.push(args) + return emptyResult() + }) +})) + +vi.mock('./skill-discovery-wsl', () => ({ + discoverSkillsInWsl: vi.fn(async (args: unknown) => { + wslScans.push(args) + return emptyResult() + }) +})) + +const { clearSkillDiscoveryCaches, discoverSkillsOnTarget } = + await import('./skill-discovery-target') +const { clearSkillRootScanCache } = await import('./discovery') + +function makeRepo(path: string): Repo { + return { + id: `repo-${path}`, + path, + displayName: 'Repo', + badgeColor: '#737373', + addedAt: 1, + kind: 'git', + connectionId: null + } +} + +beforeEach(() => { + clearSkillDiscoveryCaches() + nativeScans.length = 0 + wslScans.length = 0 + vi.mocked(clearSkillRootScanCache).mockClear() +}) + +afterEach(() => { + clearSkillDiscoveryCaches() +}) + +describe('discoverSkillsOnTarget', () => { + it('collapses simultaneous identical requests from several clients into one scan', async () => { + await Promise.all( + Array.from({ length: 12 }, () => + discoverSkillsOnTarget({ kind: 'native-host', cwd: '/workspace' }, []) + ) + ) + + expect(nativeScans).toHaveLength(1) + }) + + it('does not let two workspaces share a scan', async () => { + await Promise.all([ + discoverSkillsOnTarget({ kind: 'native-host', cwd: '/workspace-a' }, []), + discoverSkillsOnTarget({ kind: 'native-host', cwd: '/workspace-b' }, []) + ]) + + expect(nativeScans).toEqual([ + { repos: [], cwd: '/workspace-a', refresh: false }, + { repos: [], cwd: '/workspace-b', refresh: false } + ]) + }) + + it('does not let two repo lists share a scan', async () => { + await Promise.all([ + discoverSkillsOnTarget({ kind: 'native-host', cwd: undefined }, [makeRepo('/repo-a')]), + discoverSkillsOnTarget({ kind: 'native-host', cwd: undefined }, [makeRepo('/repo-b')]) + ]) + + expect(nativeScans).toHaveLength(2) + }) + + // Why: two clients can hold the same repo set in a different stored order. If the + // digest is order-sensitive they get different keys and each runs a full native + // scan — the fan-out this cache exists to bound. ttl is 0 here, so the pin is + // that two *concurrent* callers coalesce, which they only do on an equal key. + it('keeps repo-list identity stable regardless of stored order', async () => { + const repos = [makeRepo('/repo-a'), makeRepo('/repo-b')] + await discoverSkillsOnTarget({ kind: 'native-host', cwd: undefined }, repos) + await Promise.all([ + discoverSkillsOnTarget({ kind: 'native-host', cwd: undefined }, repos.toReversed()), + discoverSkillsOnTarget({ kind: 'native-host', cwd: undefined }, repos) + ]) + + expect(nativeScans).toHaveLength(2) + }) + + it('forwards refresh to the native scan so it re-reads disk', async () => { + await discoverSkillsOnTarget({ kind: 'native-host', cwd: '/workspace' }, [], { refresh: true }) + + expect(nativeScans).toEqual([{ repos: [], cwd: '/workspace', refresh: true }]) + }) + + it('reuses a WSL result rather than booting wsl.exe again', async () => { + const target = { + kind: 'wsl', + distro: 'Ubuntu', + homeDir: '/home/dev', + cwd: '/home/dev/repo' + } as const + + await discoverSkillsOnTarget(target, []) + await discoverSkillsOnTarget(target, []) + + expect(wslScans).toHaveLength(1) + }) + + it('never shares a WSL result across distros or workspaces', async () => { + await discoverSkillsOnTarget( + { kind: 'wsl', distro: 'Ubuntu', homeDir: '/home/dev', cwd: '/home/dev/a' }, + [] + ) + await discoverSkillsOnTarget( + { kind: 'wsl', distro: 'Fedora', homeDir: '/home/dev', cwd: '/home/dev/a' }, + [] + ) + await discoverSkillsOnTarget( + { kind: 'wsl', distro: 'Ubuntu', homeDir: '/home/dev', cwd: '/home/dev/b' }, + [] + ) + + expect(wslScans).toHaveLength(3) + }) + + it('re-reads a WSL target when the caller refreshes', async () => { + const target = { + kind: 'wsl', + distro: 'Ubuntu', + homeDir: '/home/dev', + cwd: '/home/dev/repo' + } as const + + await discoverSkillsOnTarget(target, []) + await discoverSkillsOnTarget(target, [], { refresh: true }) + await discoverSkillsOnTarget(target, []) + + // The refreshed result replaces the cached one, so the third call is free. + expect(wslScans).toHaveLength(2) + }) + + it('drops the root cache too when the host clears its scans', () => { + clearSkillDiscoveryCaches() + expect(clearSkillRootScanCache).toHaveBeenCalled() + }) +}) diff --git a/src/main/skills/skill-discovery-target.ts b/src/main/skills/skill-discovery-target.ts index 2be10232c8c..3670aef905e 100644 --- a/src/main/skills/skill-discovery-target.ts +++ b/src/main/skills/skill-discovery-target.ts @@ -1,8 +1,25 @@ import type { Repo } from '../../shared/types' import type { SkillDiscoveryResult, SkillDiscoveryTarget } from '../../shared/skills' import { getDefaultWslDistro, getWslHome, parseWslPath, toLinuxPath } from '../wsl' -import { discoverSkills } from './discovery' +import { clearSkillRootScanCache, discoverSkills } from './discovery' import { discoverSkillsInWsl } from './skill-discovery-wsl' +import { stablePathId } from './skill-discovery-sources' +import { getRepoExecutionHostId } from '../../shared/execution-host' +import { SkillScanCoalescer } from './skill-scan-coalescer' + +// Why: on WSL the unit of cost is the wsl.exe boot plus one `find` per skill, so +// the whole result is what must be shared. The native path shares at root level +// instead, and only needs concurrent callers collapsed into one walk. +const WSL_RESULT_TTL_MS = 10_000 +const MAX_CACHED_SKILL_TARGETS = 32 + +const targetScans = new SkillScanCoalescer(MAX_CACHED_SKILL_TARGETS) + +/** Drop every shared scan; used when a skill update run has rewritten disk. */ +export function clearSkillDiscoveryCaches(): void { + targetScans.clear() + clearSkillRootScanCache() +} export type ResolvedSkillDiscoveryTarget = | { kind: 'native-host'; cwd: string | undefined } @@ -53,18 +70,50 @@ export function resolveSkillDiscoveryTarget( return { kind: 'wsl', distro: wslDistro, homeDir: linuxHomeDir, cwd } } +// Why: repos widen the native root set, so two targets that differ only by the +// stored repo list must not share a scan. Paths are digested rather than joined +// so the key cannot grow with a large repo list. +function repoDigest(repos: readonly Repo[]): string { + return stablePathId( + repos + // Why: the source builder keeps only locally-executed repos, so the same + // path reassigned to another execution host is a different root set. + .map((repo) => `${getRepoExecutionHostId(repo)}\0${repo.path}`) + .sort((left, right) => left.localeCompare(right)) + // NUL is the one byte a path cannot contain, so no repo list can be spelled + // two ways that digest alike. + .join('\0') + ) +} + +// Keys use exact paths — lowercasing would alias two roots that are distinct on Linux. +function scanKey(target: ResolvedSkillDiscoveryTarget, repos: readonly Repo[]): string { + return target.kind === 'wsl' + ? `wsl\0${target.distro}\0${target.homeDir}\0${target.cwd}` + : `native\0${target.cwd ?? ''}\0${target.cwd ? '' : repoDigest(repos)}` +} + export async function discoverSkillsOnTarget( target: ResolvedSkillDiscoveryTarget, - repos: readonly Repo[] + repos: readonly Repo[], + options: { refresh?: boolean } = {} ): Promise { - if (target.kind === 'wsl') { - return discoverSkillsInWsl({ - distro: target.distro, - homeDir: target.homeDir, - cwd: target.cwd - }) - } - return target.cwd - ? discoverSkills({ repos: [], cwd: target.cwd }) - : discoverSkills({ repos: [...repos] }) + const refresh = options.refresh === true + const outcome = await targetScans.run( + scanKey(target, repos), + { ttlMs: target.kind === 'wsl' ? WSL_RESULT_TTL_MS : 0, refresh }, + async () => { + if (target.kind === 'wsl') { + return discoverSkillsInWsl({ + distro: target.distro, + homeDir: target.homeDir, + cwd: target.cwd + }) + } + return target.cwd + ? discoverSkills({ repos: [], cwd: target.cwd, refresh }) + : discoverSkills({ repos: [...repos], refresh }) + } + ) + return outcome.value } diff --git a/src/main/skills/skill-root-file-walk.test.ts b/src/main/skills/skill-root-file-walk.test.ts new file mode 100644 index 00000000000..654312d7f5d --- /dev/null +++ b/src/main/skills/skill-root-file-walk.test.ts @@ -0,0 +1,74 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + countPackageFiles, + findSkillFiles, + MAX_SKILL_PACKAGE_DEPTH, + MAX_SKILL_PACKAGE_FILES +} from './skill-root-file-walk' + +async function makeTree(): Promise { + return mkdtemp(join(tmpdir(), 'orca-skill-walk-')) +} + +async function writeFileAt(path: string, content = 'x'): Promise { + await mkdir(join(path, '..'), { recursive: true }) + await writeFile(path, content) +} + +describe('countPackageFiles', () => { + it('prunes node_modules so a vendored dependency tree is not the file count', async () => { + const pkg = join(await makeTree(), 'skill') + await writeFileAt(join(pkg, 'SKILL.md')) + for (let index = 0; index < 50; index += 1) { + await writeFileAt(join(pkg, 'node_modules', `dep-${index}`, 'index.js')) + } + + expect(await countPackageFiles(pkg)).toBe(1) + }) + + it('stops at the depth bound instead of walking an arbitrarily deep payload', async () => { + const pkg = join(await makeTree(), 'skill') + await writeFileAt(join(pkg, 'SKILL.md')) + const deep = join(pkg, ...Array.from({ length: MAX_SKILL_PACKAGE_DEPTH + 2 }, () => 'nested')) + await writeFileAt(join(deep, 'buried.txt')) + const reachable = join(pkg, ...Array.from({ length: MAX_SKILL_PACKAGE_DEPTH }, () => 'ok')) + await writeFileAt(join(reachable, 'shallow.txt')) + + // The reachable file is counted; the one past the bound is not. + expect(await countPackageFiles(pkg)).toBe(2) + }) + + // Why: discovery counts `dirname(SKILL.md)`, and a package can vanish between + // being found and being counted — an uninstall mid-scan is exactly that race. + it('returns zero for a directory it cannot read', async () => { + expect(await countPackageFiles(join(await makeTree(), 'absent'))).toBe(0) + }) + + it('stops at the file bound', async () => { + const pkg = join(await makeTree(), 'skill') + for (let index = 0; index < MAX_SKILL_PACKAGE_FILES + 25; index += 1) { + await writeFileAt(join(pkg, `file-${index}.txt`)) + } + + expect(await countPackageFiles(pkg)).toBe(MAX_SKILL_PACKAGE_FILES) + }) +}) + +describe('findSkillFiles', () => { + it('finds packages inside the depth bound and ignores deeper ones', async () => { + const root = join(await makeTree(), 'skills') + await writeFileAt(join(root, 'near', 'SKILL.md')) + await writeFileAt(join(root, 'a', 'b', 'c', 'd', 'far', 'SKILL.md')) + + const found = await findSkillFiles(root, 4) + + expect(found).toEqual([join(root, 'near', 'SKILL.md')]) + }) + + it('returns nothing for a missing root rather than throwing', async () => { + expect(await findSkillFiles(join(await makeTree(), 'absent'), 4)).toEqual([]) + }) +}) diff --git a/src/main/skills/skill-root-file-walk.ts b/src/main/skills/skill-root-file-walk.ts new file mode 100644 index 00000000000..ab7aedc4249 --- /dev/null +++ b/src/main/skills/skill-root-file-walk.ts @@ -0,0 +1,145 @@ +import type { Dirent } from 'node:fs' +import { readdir, realpath, stat } from 'node:fs/promises' +import { isAbsolute, join, relative, sep } from 'node:path' + +export const SKILL_FILE_NAME = 'SKILL.md' +export const MAX_SKILL_PACKAGE_FILES = 200 +// Why: `fileCount` is a display number, so the walk that produces it must not be +// able to cost more than the walk that found the skill. A package deeper than this +// under-reports its size; it never hides the skill. +export const MAX_SKILL_PACKAGE_DEPTH = 6 + +function isWithinDepth(rootPath: string, childPath: string, maxDepth: number): boolean { + const rel = relative(rootPath, childPath) + if (!rel) { + return true + } + // Why: `..cache` is a valid child name; only a real parent traversal escapes. + if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + return false + } + return rel.split(sep).length <= maxDepth +} + +async function readEntries(dirPath: string): Promise { + try { + return await readdir(dirPath, { withFileTypes: true }) + } catch { + return null + } +} + +export async function findSkillFiles(rootPath: string, maxDepth: number): Promise { + const out: string[] = [] + const visitedDirectoryPaths = new Set() + async function visit(dirPath: string): Promise { + if (!isWithinDepth(rootPath, dirPath, maxDepth)) { + return + } + let resolvedDirPath: string + try { + resolvedDirPath = await realpath(dirPath) + } catch { + return + } + if (visitedDirectoryPaths.has(resolvedDirPath)) { + return + } + visitedDirectoryPaths.add(resolvedDirPath) + + const entries = await readEntries(dirPath) + if (!entries) { + return + } + for (const entry of entries) { + const entryPath = join(dirPath, entry.name) + if (entry.name === SKILL_FILE_NAME) { + if (entry.isFile()) { + out.push(entryPath) + continue + } + if (entry.isSymbolicLink()) { + try { + if ((await stat(entryPath)).isFile()) { + out.push(entryPath) + } + } catch { + // Broken links are not valid skill files. + } + } + continue + } + if (entry.isDirectory()) { + await visit(entryPath) + continue + } + if (entry.isSymbolicLink()) { + // Why: users commonly symlink agent skill dirs across providers; follow + // directory links but guard by realpath so recursive links cannot loop. + try { + if ((await stat(entryPath)).isDirectory()) { + await visit(entryPath) + } + } catch { + // Broken links are not valid skill directories. + } + } + } + } + await visit(rootPath) + return out +} + +/** + * Files in a skill package, for the picker's size column. Bounded by count and + * depth, and `node_modules` is pruned the way the plugin cache scan prunes it — + * vendored dependencies are payload, not part of the skill. + */ +export async function countPackageFiles(dirPath: string): Promise { + let count = 0 + const visitedDirectoryPaths = new Set() + async function visit(currentPath: string, depth: number): Promise { + if (count >= MAX_SKILL_PACKAGE_FILES || depth > MAX_SKILL_PACKAGE_DEPTH) { + return + } + let resolvedPath: string + try { + resolvedPath = await realpath(currentPath) + } catch { + return + } + if (visitedDirectoryPaths.has(resolvedPath)) { + return + } + visitedDirectoryPaths.add(resolvedPath) + + const entries = await readEntries(currentPath) + if (!entries) { + return + } + for (const entry of entries) { + if (count >= MAX_SKILL_PACKAGE_FILES) { + return + } + if (entry.name === 'node_modules') { + continue + } + const entryPath = join(currentPath, entry.name) + if (entry.isFile()) { + count += 1 + } else if (entry.isDirectory()) { + await visit(entryPath, depth + 1) + } else if (entry.isSymbolicLink()) { + try { + if ((await stat(entryPath)).isFile()) { + count += 1 + } + } catch { + // Broken links do not contribute to the skill package file count. + } + } + } + } + await visit(dirPath, 0) + return count +} diff --git a/src/main/skills/skill-scan-coalescer.test.ts b/src/main/skills/skill-scan-coalescer.test.ts new file mode 100644 index 00000000000..7a8d1bb1539 --- /dev/null +++ b/src/main/skills/skill-scan-coalescer.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, it } from 'vitest' +import { SkillScanCoalescer } from './skill-scan-coalescer' + +function deferred(): { promise: Promise; resolve: (value: T) => void } { + let resolve!: (value: T) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +describe('SkillScanCoalescer', () => { + it('collapses concurrent callers on one key into a single scan', async () => { + const coalescer = new SkillScanCoalescer(8) + const gate = deferred() + let runs = 0 + const task = (): Promise => { + runs += 1 + return gate.promise + } + + const outcomes = Promise.all([ + coalescer.run('root', { ttlMs: 0 }, task), + coalescer.run('root', { ttlMs: 0 }, task), + coalescer.run('root', { ttlMs: 0 }, task) + ]) + gate.resolve(7) + + expect((await outcomes).map((outcome) => outcome.value)).toEqual([7, 7, 7]) + expect((await outcomes).map((outcome) => outcome.cached)).toEqual([false, true, true]) + expect(runs).toBe(1) + }) + + it('keeps distinct keys isolated, including paths differing only by case', async () => { + const coalescer = new SkillScanCoalescer(8) + const seen: string[] = [] + const run = (key: string): Promise<{ value: string }> => + coalescer.run(key, { ttlMs: 1_000 }, async () => { + seen.push(key) + return key + }) + + const [lower, upper] = await Promise.all([run('/home/a/Skills'), run('/home/a/skills')]) + + expect(lower.value).toBe('/home/a/Skills') + expect(upper.value).toBe('/home/a/skills') + expect(seen).toHaveLength(2) + }) + + it('reuses a result inside the ttl and rescans after it lapses', async () => { + let now = 1_000 + const coalescer = new SkillScanCoalescer(8, () => now) + let runs = 0 + const task = async (): Promise => { + runs += 1 + return runs + } + + expect((await coalescer.run('root', { ttlMs: 100 }, task)).cached).toBe(false) + now = 1_050 + const cached = await coalescer.run('root', { ttlMs: 100 }, task) + expect(cached).toEqual({ value: 1, cached: true }) + now = 1_101 + expect(await coalescer.run('root', { ttlMs: 100 }, task)).toEqual({ value: 2, cached: false }) + expect(runs).toBe(2) + }) + + it('retains nothing when the ttl is zero', async () => { + const coalescer = new SkillScanCoalescer(8) + let runs = 0 + const task = async (): Promise => { + runs += 1 + return runs + } + + await coalescer.run('root', { ttlMs: 0 }, task) + await coalescer.run('root', { ttlMs: 0 }, task) + + expect(runs).toBe(2) + }) + + it('bypasses cached and in-flight results when refreshing', async () => { + let now = 1_000 + const coalescer = new SkillScanCoalescer(8, () => now) + let runs = 0 + const task = async (): Promise => { + runs += 1 + return runs + } + + await coalescer.run('root', { ttlMs: 10_000 }, task) + const refreshed = await coalescer.run('root', { ttlMs: 10_000, refresh: true }, task) + + expect(refreshed).toEqual({ value: 2, cached: false }) + // The refreshed result is what later readers see, not the entry it replaced. + expect(await coalescer.run('root', { ttlMs: 10_000 }, task)).toEqual({ value: 2, cached: true }) + expect(runs).toBe(2) + }) + + // Why: discovery issues one refreshing run() per root, synchronously. An + // invalidation counter shared across keys would let only the last-issued root + // publish and silently discard the rest, so every later scan re-walks them. + it('caches every key when a refresh fans out across roots', async () => { + const coalescer = new SkillScanCoalescer(64) + const keys = ['root-a', 'root-b', 'root-c', 'root-d'] + + await Promise.all( + keys.map((key) => coalescer.run(key, { ttlMs: 10_000, refresh: true }, async () => key)) + ) + const readBack = await Promise.all( + keys.map((key) => coalescer.run(key, { ttlMs: 10_000 }, async () => 're-walked')) + ) + + expect(readBack.map((outcome) => outcome.cached)).toEqual([true, true, true, true]) + expect(readBack.map((outcome) => outcome.value)).toEqual(keys) + }) + + it('does not let a scan that started before a refresh publish its stale result', async () => { + const coalescer = new SkillScanCoalescer(8) + const slowScan = deferred() + + // A focus/mount scan is already in flight when the user installs a skill. + const inFlight = coalescer.run('root', { ttlMs: 10_000 }, () => slowScan.promise) + const refreshed = await coalescer.run( + 'root', + { ttlMs: 10_000, refresh: true }, + async () => 'after-install' + ) + expect(refreshed.value).toBe('after-install') + + // The older scan now lands. It must not overwrite the post-install entry with + // a pre-install listing and a fresh lifetime. + slowScan.resolve('before-install') + await inFlight + + expect(await coalescer.run('root', { ttlMs: 10_000 }, async () => 'rescanned')).toEqual({ + value: 'after-install', + cached: true + }) + }) + + it('does not let a scan that started before clear() repopulate the cache', async () => { + const coalescer = new SkillScanCoalescer(8) + const slowScan = deferred() + + const inFlight = coalescer.run('root', { ttlMs: 10_000 }, () => slowScan.promise) + // A skill update run rewrote disk while that scan was walking it. + coalescer.clear() + slowScan.resolve('pre-update') + await inFlight + + expect(await coalescer.run('root', { ttlMs: 10_000 }, async () => 'post-update')).toEqual({ + value: 'post-update', + cached: false + }) + }) + + it('does not cache a failed scan', async () => { + const coalescer = new SkillScanCoalescer(8) + let runs = 0 + + await expect( + coalescer.run('root', { ttlMs: 10_000 }, async () => { + runs += 1 + throw new Error('scan failed') + }) + ).rejects.toThrow('scan failed') + expect(await coalescer.run('root', { ttlMs: 10_000 }, async () => 5)).toEqual({ + value: 5, + cached: false + }) + expect(runs).toBe(1) + }) + + it('evicts the least recently used entry past the bound', async () => { + let now = 1_000 + const coalescer = new SkillScanCoalescer(2, () => now) + const scan = (key: string): Promise<{ cached: boolean }> => + coalescer.run(key, { ttlMs: 10_000 }, async () => key) + + await scan('a') + await scan('b') + // Reading 'a' promotes it, so 'b' is the eviction candidate when 'c' arrives. + expect((await scan('a')).cached).toBe(true) + await scan('c') + + expect((await scan('a')).cached).toBe(true) + expect((await scan('b')).cached).toBe(false) + }) + + // ttl is non-zero on purpose: with ttl 0 nothing is ever written, which hides + // whether the abandoned scan can still publish over the replacement's result. + it('stops joining a scan that never settles, and never lets it publish later', async () => { + let now = 1_000 + const coalescer = new SkillScanCoalescer(8, () => now) + const wedged = deferred() + let runs = 0 + const task = (): Promise => { + runs += 1 + // The first scan models a root on a stalled mount: its readdir never settles. + return runs === 1 ? wedged.promise : Promise.resolve(runs) + } + + void coalescer.run('root', { ttlMs: 10_000 }, task) + now = 1_100 + // Still young enough to share. + const joined = coalescer.run('root', { ttlMs: 10_000 }, task) + now = 40_000 + + expect(await coalescer.run('root', { ttlMs: 10_000 }, task)).toEqual({ + value: 2, + cached: false + }) + expect(runs).toBe(2) + + // The wedged callers still receive its eventual value rather than being orphaned... + wedged.resolve(99) + expect((await joined).value).toBe(99) + // ...but it must not overwrite the replacement's newer result with a fresh ttl. + expect(await coalescer.run('root', { ttlMs: 10_000 }, task)).toEqual({ value: 2, cached: true }) + }) +}) diff --git a/src/main/skills/skill-scan-coalescer.ts b/src/main/skills/skill-scan-coalescer.ts new file mode 100644 index 00000000000..422a3045783 --- /dev/null +++ b/src/main/skills/skill-scan-coalescer.ts @@ -0,0 +1,121 @@ +/** `cached` is false when this call did the filesystem work, for diagnostics. */ +export type SkillScanOutcome = { value: T; cached: boolean } + +export type SkillScanRunOptions = { + /** 0 keeps nothing after the scan settles, so the entry only dedups concurrent callers. */ + ttlMs: number + /** Skip every cached and in-flight result and re-read disk. */ + refresh?: boolean +} + +type CacheEntry = { value: T; expiresAt: number } +type PendingEntry = { promise: Promise; startedAt: number } + +// Why: a root on a stalled network mount can leave a readdir that never settles. +// Joining it forever would make one wedged mount permanently wedge discovery for +// every later caller — worse than before this cache existed, where each caller at +// least retried. Past this age a new caller starts its own scan instead; the old +// promise is dropped, so at most one pending entry per key survives. +const MAX_JOINABLE_SCAN_AGE_MS = 30_000 + +/** + * Shares one filesystem scan between concurrent callers and, optionally, reuses + * its result for a short window. + * + * Keys are used verbatim — callers must not normalize case, because two paths + * that differ only by case can be two different targets on Linux. + */ +export class SkillScanCoalescer { + private readonly pending = new Map>() + private readonly cache = new Map>() + + constructor( + private readonly maximumEntries: number, + private readonly now: () => number = Date.now + ) {} + + async run( + key: string, + options: SkillScanRunOptions, + task: () => Promise + ): Promise> { + if (options.refresh) { + // Why: a forced caller is answering a mutation it just made, so it must not + // join a scan that may have started before that mutation. Concurrent forced + // callers therefore duplicate; they are rare (install / explicit recheck). + this.cache.delete(key) + return { value: await this.start(key, options.ttlMs, task), cached: false } + } + const fresh = this.readFresh(key) + if (fresh) { + return { value: fresh.value, cached: true } + } + const inFlight = this.pending.get(key) + if (inFlight && this.now() - inFlight.startedAt < MAX_JOINABLE_SCAN_AGE_MS) { + return { value: await inFlight.promise, cached: true } + } + return { value: await this.start(key, options.ttlMs, task), cached: false } + } + + /** Drop every cached and in-flight entry (e.g. after a skill update run). */ + clear(): void { + this.cache.clear() + this.pending.clear() + } + + private start(key: string, ttlMs: number, task: () => Promise): Promise { + const promise = task() + .then((value) => { + // Why: owning the pending slot is what makes a scan publishable, and it is + // per key by construction. Deleting the cache entry is not enough to + // invalidate — a scan that began before the mutation resolves afterwards + // and would re-cache its pre-mutation result with a fresh lifetime. This + // one check covers all three ways it can be superseded: `clear()` empties + // `pending`, a refresh overwrites the slot, and so does the replacement + // for a scan abandoned past MAX_JOINABLE_SCAN_AGE_MS. + if (ttlMs > 0 && this.pending.get(key)?.promise === promise) { + this.write(key, value, ttlMs) + } + return value + }) + .finally(() => { + // Why: a newer forced scan may already own this key; only the entry that + // registered itself may remove itself. + if (this.pending.get(key)?.promise === promise) { + this.pending.delete(key) + } + }) + // Why: rejections must not surface as an unhandled rejection on the shared + // promise before the caller that started it awaits. + promise.catch(() => undefined) + this.pending.set(key, { promise, startedAt: this.now() }) + return promise + } + + private readFresh(key: string): CacheEntry | null { + const entry = this.cache.get(key) + if (!entry) { + return null + } + if (entry.expiresAt <= this.now()) { + this.cache.delete(key) + return null + } + // Refresh recency so a hot root outlives a one-off target under the bound. + this.cache.delete(key) + this.cache.set(key, entry) + return entry + } + + private write(key: string, value: T, ttlMs: number): void { + this.cache.delete(key) + this.cache.set(key, { value, expiresAt: this.now() + ttlMs }) + while (this.cache.size > this.maximumEntries) { + const oldestKey = this.cache.keys().next().value + if (oldestKey === undefined) { + break + } + this.cache.delete(oldestKey) + } + } +} diff --git a/src/renderer/src/components/native-chat/use-native-chat-skills.react.test.tsx b/src/renderer/src/components/native-chat/use-native-chat-skills.react.test.tsx index 647a07b2f4f..8004071d679 100644 --- a/src/renderer/src/components/native-chat/use-native-chat-skills.react.test.tsx +++ b/src/renderer/src/components/native-chat/use-native-chat-skills.react.test.tsx @@ -1,6 +1,6 @@ // @vitest-environment happy-dom -import { cleanup, render, waitFor } from '@testing-library/react' +import { act, cleanup, render, waitFor } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { NativeChatSkillDiscovery } from './use-native-chat-skills' @@ -130,6 +130,31 @@ describe('useNativeChatSkills', () => { expect(mocks.callRuntimeRpc).not.toHaveBeenCalled() }) + it('makes Retry reach disk instead of the host shared scan', async () => { + render() + await waitFor(() => expect(mocks.snapshots.at(-1)?.status).toBe('ready')) + expect(mocks.callRuntimeRpc).toHaveBeenLastCalledWith( + { kind: 'local' }, + 'skills.discover', + { cwd: '/repo/worktree', worktreeId: 'worktree-1' }, + { timeoutMs: 10_000 } + ) + + act(() => { + mocks.snapshots.at(-1)?.retry() + }) + await waitFor(() => expect(mocks.callRuntimeRpc).toHaveBeenCalledTimes(2)) + + // Why: Retry is the user saying "I changed something" — without `refresh` it + // would be answered from the scan it is trying to get past. + expect(mocks.callRuntimeRpc).toHaveBeenLastCalledWith( + { kind: 'local' }, + 'skills.discover', + { cwd: '/repo/worktree', worktreeId: 'worktree-1', refresh: true }, + { timeoutMs: 10_000 } + ) + }) + it('routes runtime-owned panes through their saved environment', async () => { mocks.state = stateForHost('runtime:env-1') render() diff --git a/src/renderer/src/components/native-chat/use-native-chat-skills.ts b/src/renderer/src/components/native-chat/use-native-chat-skills.ts index 42c7702d017..f8a7e943935 100644 --- a/src/renderer/src/components/native-chat/use-native-chat-skills.ts +++ b/src/renderer/src/components/native-chat/use-native-chat-skills.ts @@ -83,11 +83,17 @@ export function useNativeChatSkills( const [state, setState] = useState(IDLE_STATE) const [retryGeneration, setRetryGeneration] = useState(0) const paneDiscoveryCache = useRef(new Map()) + // Why: retry intent belongs to the next request, not to every later render — + // keying off the generation counter would keep forcing after a pane switch. + const forceNextDiscovery = useRef(false) const profile = getNativeChatAgentProfile(agent) useEffect(() => { let cancelled = false if (!profile || !enabled || !context) { + // Why: there is no pane to retry into, so a pending retry intent must not + // survive to force an unrelated pane's first scan. + forceNextDiscovery.current = false setState(IDLE_STATE) return } @@ -119,7 +125,12 @@ export function useNativeChatSkills( return } setState({ status: 'loading', skills: [], error: null, contextKey: context.key }) - const request = getOrStartDiscovery(context) + // Why: Retry is an explicit "I changed something, look again", so it has to + // reach the host's disk rather than its shared scans. The first attempt for a + // pane rides those scans like every other passive reader. + const forced = forceNextDiscovery.current + forceNextDiscovery.current = false + const request = getOrStartDiscovery(context, forced) void request.then( (result) => { paneDiscoveryCache.current.set(paneCacheKey, result) @@ -182,6 +193,7 @@ export function useNativeChatSkills( }, [agent, context, effectiveState, profile]) const retry = useCallback(() => { + forceNextDiscovery.current = true if (context) { paneDiscoveryCache.current.delete(context.key) setState({ status: 'loading', skills: [], error: null, contextKey: context.key }) @@ -201,10 +213,11 @@ export function useNativeChatSkills( } function getOrStartDiscovery( - context: NativeChatSkillDiscoveryContext + context: NativeChatSkillDiscoveryContext, + refresh = false ): Promise { const existing = inFlightDiscovery.get(context.key) - if (existing) { + if (existing && !refresh) { return existing } // Why: the local runtime.call branch ignores timeoutMs, so the renderer must @@ -213,7 +226,7 @@ function getOrStartDiscovery( callRuntimeRpc( context.runtimeTarget, 'skills.discover', - context.discoveryTarget, + refresh ? { ...context.discoveryTarget, refresh: true } : context.discoveryTarget, { timeoutMs: DISCOVERY_TIMEOUT_MS } diff --git a/src/renderer/src/components/skills/SkillsPage.tsx b/src/renderer/src/components/skills/SkillsPage.tsx index 70b7795912b..8bb3c6fef9d 100644 --- a/src/renderer/src/components/skills/SkillsPage.tsx +++ b/src/renderer/src/components/skills/SkillsPage.tsx @@ -85,36 +85,44 @@ export default function SkillsPage(): React.JSX.Element { const mountedRef = useMountedRef() const scanGenerationRef = useRef(0) - const loadSkills = useCallback(async (): Promise => { - setLoading(true) - // Why: a cold local scan walks every skill root, so switching runtimes can - // land a stale result after a newer one. Only the newest scan may write. - const scanGeneration = ++scanGenerationRef.current - const isCurrentScan = (): boolean => - mountedRef.current && scanGeneration === scanGenerationRef.current - if (!runtimeTarget) { - // Why: keep scanning until the owning runtime is known, rather than - // showing the client's skills to someone whose skills live remotely. - return - } - try { - const nextResult = await discoverSkillsForRuntimeTarget(runtimeTarget) - if (isCurrentScan()) { - setResult(nextResult) + // `refresh` bypasses the host's shared scans; only an explicit user re-scan does + // that, so opening the page rides the same cached roots every other surface uses. + const loadSkills = useCallback( + async (refresh = false): Promise => { + setLoading(true) + // Why: a cold local scan walks every skill root, so switching runtimes can + // land a stale result after a newer one. Only the newest scan may write. + const scanGeneration = ++scanGenerationRef.current + const isCurrentScan = (): boolean => + mountedRef.current && scanGeneration === scanGenerationRef.current + if (!runtimeTarget) { + // Why: keep scanning until the owning runtime is known, rather than + // showing the client's skills to someone whose skills live remotely. + return } - } catch (error) { - console.error('Failed to discover skills:', error) - if (isCurrentScan()) { - toast.error( - translate('auto.components.skills.SkillsPage.ea72d6185b', 'Could not scan skills') + try { + const nextResult = await discoverSkillsForRuntimeTarget( + runtimeTarget, + refresh ? { refresh: true } : undefined ) + if (isCurrentScan()) { + setResult(nextResult) + } + } catch (error) { + console.error('Failed to discover skills:', error) + if (isCurrentScan()) { + toast.error( + translate('auto.components.skills.SkillsPage.ea72d6185b', 'Could not scan skills') + ) + } + } finally { + if (isCurrentScan()) { + setLoading(false) + } } - } finally { - if (isCurrentScan()) { - setLoading(false) - } - } - }, [mountedRef, runtimeTarget]) + }, + [mountedRef, runtimeTarget] + ) useEffect(() => { void loadSkills() @@ -274,7 +282,7 @@ export default function SkillsPage(): React.JSX.Element { className="h-8" disabled={loading} onClick={() => { - void loadSkills() + void loadSkills(true) }} > @@ -302,7 +310,7 @@ export default function SkillsPage(): React.JSX.Element { 0} - onRefresh={() => void loadSkills()} + onRefresh={() => void loadSkills(true)} /> )} diff --git a/src/renderer/src/hooks/installed-agent-skill-discovery-cache.test.ts b/src/renderer/src/hooks/installed-agent-skill-discovery-cache.test.ts index ed23b3c5d76..0a99ddd26df 100644 --- a/src/renderer/src/hooks/installed-agent-skill-discovery-cache.test.ts +++ b/src/renderer/src/hooks/installed-agent-skill-discovery-cache.test.ts @@ -1,10 +1,11 @@ -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import type { SkillDiscoveryResult } from '../../../shared/skills' import { clearInstalledAgentSkillDiscoveryCache, getInstalledAgentSkillDiscoveryCacheSizeForTests, hasInstalledAgentSkillDiscoveryCacheEntryForTests, INSTALLED_AGENT_SKILL_DISCOVERY_CACHE_MAX, + INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS, peekInstalledAgentSkillDiscoveryCache, readInstalledAgentSkillDiscoveryCache, resetInstalledAgentSkillDiscoveryCacheForTests, @@ -17,6 +18,7 @@ function result(scannedAt: number): SkillDiscoveryResult { afterEach(() => { resetInstalledAgentSkillDiscoveryCacheForTests() + vi.restoreAllMocks() }) describe('installed agent skill discovery cache', () => { @@ -89,4 +91,28 @@ describe('installed agent skill discovery cache', () => { expect(getInstalledAgentSkillDiscoveryCacheSizeForTests()).toBe(0) expect(peekInstalledAgentSkillDiscoveryCache('target')).toBeNull() }) + + it('stops serving a read once the freshness window lapses', () => { + const startedAt = 1_700_000_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(startedAt) + writeInstalledAgentSkillDiscoveryCache('target', result(1)) + + nowSpy.mockReturnValue(startedAt + INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS - 1) + expect(readInstalledAgentSkillDiscoveryCache('target')).toEqual(result(1)) + + nowSpy.mockReturnValue(startedAt + INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS) + expect(readInstalledAgentSkillDiscoveryCache('target')).toBeNull() + expect(getInstalledAgentSkillDiscoveryCacheSizeForTests()).toBe(0) + }) + + it('still peeks a lapsed result so a first render shows the last known state', () => { + const startedAt = 1_700_000_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(startedAt) + writeInstalledAgentSkillDiscoveryCache('target', result(1)) + + nowSpy.mockReturnValue(startedAt + INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS + 1) + + // A lapsed entry triggers a rescan; showing it beats flashing empty meanwhile. + expect(peekInstalledAgentSkillDiscoveryCache('target')).toEqual(result(1)) + }) }) diff --git a/src/renderer/src/hooks/installed-agent-skill-discovery-cache.ts b/src/renderer/src/hooks/installed-agent-skill-discovery-cache.ts index a93abe2c3f9..29d30eb4c98 100644 --- a/src/renderer/src/hooks/installed-agent-skill-discovery-cache.ts +++ b/src/renderer/src/hooks/installed-agent-skill-discovery-cache.ts @@ -5,21 +5,44 @@ import type { SkillDiscoveryResult } from '../../../shared/skills' // the window has ever resolved. export const INSTALLED_AGENT_SKILL_DISCOVERY_CACHE_MAX = 256 -let cachedDiscoveryByTarget = new Map() +// Why: focus-triggered refreshes read through this cache instead of forcing a +// disk walk, so it needs a lifetime — without one a non-forced read would never +// see a skill installed outside Orca. Matches the focus-rescan cooldown the +// freshness inventory already applies to its own scan (`useSkillFreshness`), so +// the two disk-reading surfaces answer a burst of alt-tabs the same way. +export const INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS = 15_000 + +type CachedDiscovery = { result: SkillDiscoveryResult; expiresAt: number } + +let cachedDiscoveryByTarget = new Map() + +function readUnexpired(key: string): CachedDiscovery | null { + const cached = cachedDiscoveryByTarget.get(key) + if (!cached) { + return null + } + if (cached.expiresAt <= Date.now()) { + cachedDiscoveryByTarget.delete(key) + return null + } + return cached +} // Why: render reads must not reorder recency — React can discard a render pass. export function peekInstalledAgentSkillDiscoveryCache(key: string): SkillDiscoveryResult | null { - return cachedDiscoveryByTarget.get(key) ?? null + // Why: an expired entry is still the last thing this window knew, and showing it + // beats flashing empty while the refresh it triggers is in flight. + return cachedDiscoveryByTarget.get(key)?.result ?? null } export function readInstalledAgentSkillDiscoveryCache(key: string): SkillDiscoveryResult | null { - const result = cachedDiscoveryByTarget.get(key) - if (!result) { + const cached = readUnexpired(key) + if (!cached) { return null } cachedDiscoveryByTarget.delete(key) - cachedDiscoveryByTarget.set(key, result) - return result + cachedDiscoveryByTarget.set(key, cached) + return cached.result } export function writeInstalledAgentSkillDiscoveryCache( @@ -27,7 +50,10 @@ export function writeInstalledAgentSkillDiscoveryCache( result: SkillDiscoveryResult ): void { cachedDiscoveryByTarget.delete(key) - cachedDiscoveryByTarget.set(key, result) + cachedDiscoveryByTarget.set(key, { + result, + expiresAt: Date.now() + INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS + }) while (cachedDiscoveryByTarget.size > INSTALLED_AGENT_SKILL_DISCOVERY_CACHE_MAX) { const oldestKey = cachedDiscoveryByTarget.keys().next().value if (oldestKey === undefined) { diff --git a/src/renderer/src/hooks/installed-agent-skill-discovery.ts b/src/renderer/src/hooks/installed-agent-skill-discovery.ts index a501611889c..2530bdeb11a 100644 --- a/src/renderer/src/hooks/installed-agent-skill-discovery.ts +++ b/src/renderer/src/hooks/installed-agent-skill-discovery.ts @@ -104,7 +104,10 @@ function startInstalledAgentSkillDiscovery( ): Promise { const generation = discoveryGeneration const normalizedTarget = normalizeSkillDiscoveryTarget(target) - const discovery = discoverSkillsForRuntimeTarget(runtimeTarget, normalizedTarget) + // Why: a forced caller knows disk changed (install finished, explicit recheck), + // so it must also bypass the host's shared scans — not just this window's cache. + const requestTarget = force ? { ...normalizedTarget, refresh: true } : normalizedTarget + const discovery = discoverSkillsForRuntimeTarget(runtimeTarget, requestTarget) .then((result) => { if (generation === discoveryGeneration) { writeInstalledAgentSkillDiscoveryCache(key, result) diff --git a/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx b/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx index f6b446964ac..19494e46aac 100644 --- a/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx +++ b/src/renderer/src/hooks/useInstalledAgentSkills.react.test.tsx @@ -13,6 +13,7 @@ import type { GlobalSettings } from '../../../shared/types' import { createCompatibleRuntimeStatusResponseIfNeeded } from '@/runtime/runtime-compatibility-test-fixture' import { clearRuntimeCompatibilityCacheForTests } from '@/runtime/runtime-rpc-client' import { useAppStore } from '@/store' +import { INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS } from './installed-agent-skill-discovery-cache' import { GLOBAL_AGENT_SKILL_SOURCE_KINDS, type InstalledAgentSkillState, @@ -216,7 +217,8 @@ describe('useInstalledAgentSkill', () => { expect(latestState?.installed).toBe(false) expect(discover).toHaveBeenNthCalledWith(1, undefined) - expect(discover).toHaveBeenNthCalledWith(2, undefined) + // A forced refresh must also bypass the host's shared scans, not just this cache. + expect(discover).toHaveBeenNthCalledWith(2, { refresh: true }) }) it('returns installed from refresh when a legacy Linear skill is discovered', async () => { @@ -289,13 +291,15 @@ describe('useInstalledAgentSkill', () => { }) }) - it('stays settled through a focus rescan so status surfaces do not flash', async () => { + it('serves a focus rescan from cache so window switching does not walk disk', async () => { + // Why: the freshness window is wall-clock, so pin the clock — a stalled runner + // could otherwise cross it mid-test and turn this into a flake. + vi.spyOn(Date, 'now').mockReturnValue(1_700_000_000_000) const firstScan = deferred() - const focusScan = deferred() const discover = vi .fn<(target?: SkillDiscoveryTarget) => Promise>() .mockReturnValueOnce(firstScan.promise) - .mockReturnValueOnce(focusScan.promise) + .mockResolvedValue(discoveryResult([skill({ name: 'orca-linear' })])) Object.defineProperty(window, 'api', { configurable: true, value: { skills: { discover } } @@ -310,21 +314,48 @@ describe('useInstalledAgentSkill', () => { }) expect(latestState?.settled).toBe(true) expect(latestState?.installed).toBe(true) + expect(discover).toHaveBeenCalledTimes(1) + for (let focusEvent = 0; focusEvent < 5; focusEvent += 1) { + await act(async () => { + window.dispatchEvent(new Event('focus')) + }) + await flushMicrotasks() + } + + // Focus is a backstop, not a mutation signal: a burst of window switches + // costs no scans, and the surface never flashes unsettled. + expect(discover).toHaveBeenCalledTimes(1) + expect(latestState?.loading).toBe(false) + expect(latestState?.settled).toBe(true) + expect(latestState?.installed).toBe(true) + }) + + it('rescans on focus once the cached scan is no longer fresh', async () => { + const nowSpy = vi.spyOn(Date, 'now') + const startedAt = 1_700_000_000_000 + nowSpy.mockReturnValue(startedAt) + const discover = vi + .fn<(target?: SkillDiscoveryTarget) => Promise>() + .mockResolvedValue(discoveryResult([skill({ name: 'orca-linear' })])) + Object.defineProperty(window, 'api', { + configurable: true, + value: { skills: { discover } } + }) + + await renderProbe() + await flushMicrotasks() + expect(discover).toHaveBeenCalledTimes(1) + + nowSpy.mockReturnValue(startedAt + INSTALLED_AGENT_SKILL_DISCOVERY_FRESH_MS + 1) await act(async () => { window.dispatchEvent(new Event('focus')) }) + await flushMicrotasks() - // The forced rescan is in flight, but the previous result is still current. - expect(latestState?.loading).toBe(true) - expect(latestState?.settled).toBe(true) - expect(latestState?.installed).toBe(true) - - focusScan.resolve(discoveryResult([skill({ name: 'orca-linear' })])) - await act(async () => { - await focusScan.promise - }) - expect(latestState?.settled).toBe(true) + // The freshness window is what bounds the storm; past it, focus still reads disk. + expect(discover).toHaveBeenCalledTimes(2) + expect(discover).toHaveBeenLastCalledWith(undefined) }) it('reuses cached discovery when another surface finishes re-checking', async () => { @@ -351,13 +382,13 @@ describe('useInstalledAgentSkill', () => { expect(latestState?.installed).toBe(true) }) - it('clears loading when a silent refresh supersedes an in-flight focus rescan', async () => { + it('clears loading when a silent refresh supersedes an in-flight forced rescan', async () => { const firstScan = deferred() - const focusScan = deferred() + const forcedScan = deferred() const discover = vi .fn<(target?: SkillDiscoveryTarget) => Promise>() .mockReturnValueOnce(firstScan.promise) - .mockReturnValueOnce(focusScan.promise) + .mockReturnValueOnce(forcedScan.promise) Object.defineProperty(window, 'api', { configurable: true, value: { skills: { discover } } @@ -369,8 +400,10 @@ describe('useInstalledAgentSkill', () => { await firstScan.promise }) + // The recheck button forces past the cache without clearing it, so the silent + // refresh below can still be served from it while this scan is in flight. await act(async () => { - window.dispatchEvent(new Event('focus')) + void latestState?.refresh() }) expect(latestState?.loading).toBe(true) @@ -382,9 +415,9 @@ describe('useInstalledAgentSkill', () => { await flushMicrotasks() expect(latestState?.loading).toBe(false) - focusScan.resolve(discoveryResult([skill({ name: 'orca-linear' })])) + forcedScan.resolve(discoveryResult([skill({ name: 'orca-linear' })])) await act(async () => { - await focusScan.promise + await forcedScan.promise }) expect(latestState?.loading).toBe(false) }) diff --git a/src/renderer/src/hooks/useInstalledAgentSkills.ts b/src/renderer/src/hooks/useInstalledAgentSkills.ts index 482ea67386d..a7d96fe9451 100644 --- a/src/renderer/src/hooks/useInstalledAgentSkills.ts +++ b/src/renderer/src/hooks/useInstalledAgentSkills.ts @@ -247,21 +247,26 @@ export function useInstalledAgentSkillNames( if (!enabled) { return } - const refreshFromExternalChange = (): void => { + // Why: skill install commands run outside React state, often in a terminal, so + // an install event is authoritative and forces past every cache. + const refreshFromInstall = (): void => { void refresh(true) } - const refreshFromCompletedScan = (): void => { + // Why: focus fires on every app and window switch, and a forced refresh + // bypasses every cache down to the host's disk walk — that is what turned an + // alt-tab into a multi-root filesystem scan per window and per client. Focus, + // and another surface finishing its own scan, are both only hints that + // something may have changed, so they read through the freshness window. + const refreshQuietly = (): void => { void refresh(false, false) } - // Why: skill install commands run outside React state, often in a terminal. - // Refresh on focus and explicit install events so completion is detected. - window.addEventListener('focus', refreshFromExternalChange) - window.addEventListener(INSTALLED_AGENT_SKILLS_CHANGED_EVENT, refreshFromExternalChange) - window.addEventListener(INSTALLED_AGENT_SKILLS_REFRESHED_EVENT, refreshFromCompletedScan) + window.addEventListener('focus', refreshQuietly) + window.addEventListener(INSTALLED_AGENT_SKILLS_CHANGED_EVENT, refreshFromInstall) + window.addEventListener(INSTALLED_AGENT_SKILLS_REFRESHED_EVENT, refreshQuietly) return () => { - window.removeEventListener('focus', refreshFromExternalChange) - window.removeEventListener(INSTALLED_AGENT_SKILLS_CHANGED_EVENT, refreshFromExternalChange) - window.removeEventListener(INSTALLED_AGENT_SKILLS_REFRESHED_EVENT, refreshFromCompletedScan) + window.removeEventListener('focus', refreshQuietly) + window.removeEventListener(INSTALLED_AGENT_SKILLS_CHANGED_EVENT, refreshFromInstall) + window.removeEventListener(INSTALLED_AGENT_SKILLS_REFRESHED_EVENT, refreshQuietly) } }, [enabled, refresh]) diff --git a/src/renderer/src/runtime/runtime-skills-client.test.ts b/src/renderer/src/runtime/runtime-skills-client.test.ts index 197db33cddf..e4535e3eff6 100644 --- a/src/renderer/src/runtime/runtime-skills-client.test.ts +++ b/src/renderer/src/runtime/runtime-skills-client.test.ts @@ -99,4 +99,24 @@ describe('discoverSkillsForRuntimeTarget', () => { expect.objectContaining({ method: 'skills.discover', params: {} }) ) }) + + // Why: refresh describes the request, not the client's host. Dropping it would + // leave an explicit re-check reading the remote host's shared scan instead of + // its disk, which is exactly what an install-completed refresh must not do. + it('forwards an explicit refresh to a remote runtime', async () => { + runtimeEnvironmentCall.mockResolvedValueOnce({ + id: 'skills', + ok: true, + result: discoveryResult('orchestration') + }) + + await discoverSkillsForRuntimeTarget( + { kind: 'environment', environmentId: 'env-1' }, + { runtime: 'wsl', wslDistro: 'Ubuntu', refresh: true } + ) + + expect(runtimeEnvironmentCall).toHaveBeenCalledWith( + expect.objectContaining({ method: 'skills.discover', params: { refresh: true } }) + ) + }) }) diff --git a/src/renderer/src/runtime/runtime-skills-client.ts b/src/renderer/src/runtime/runtime-skills-client.ts index 700e90a66fd..40101379a81 100644 --- a/src/renderer/src/runtime/runtime-skills-client.ts +++ b/src/renderer/src/runtime/runtime-skills-client.ts @@ -9,13 +9,17 @@ const SKILL_DISCOVERY_TIMEOUT_MS = 15_000 * when one is active. This keeps install badges in sync with where the skill * files land instead of always reading the client's disk (#6789). * - * The target is dropped entirely for a remote call. Every target any caller can + * The target is otherwise dropped for a remote call. Every target any caller can * currently produce describes the *client's* host — a WSL distro or a local * project-runtime resolution — and forwarding those would ask a Linux server to * resolve a WSL distro it does not have. The server does honour `cwd` and * `worktreeId` (see `main/runtime/rpc/methods/skills.ts`), so if a caller ever * supplies workspace identity, forward those two fields rather than widening * this to the whole target. + * + * `refresh` is the exception and must be forwarded: it describes the *request*, + * not the client's host, and it is the only way an explicit re-check reaches + * past the remote host's shared scans to its disk. */ export async function discoverSkillsForRuntimeTarget( runtimeTarget: RuntimeClientTarget, @@ -27,7 +31,7 @@ export async function discoverSkillsForRuntimeTarget( return callRuntimeRpc( runtimeTarget, 'skills.discover', - {}, + target?.refresh ? { refresh: true } : {}, { timeoutMs: SKILL_DISCOVERY_TIMEOUT_MS } ) } diff --git a/src/shared/skills.ts b/src/shared/skills.ts index c908954d3a9..381ebf6eb3a 100644 --- a/src/shared/skills.ts +++ b/src/shared/skills.ts @@ -51,6 +51,9 @@ export type SkillDiscoveryTarget = { * when the caller (e.g. a remote client) cannot supply `projectRuntime`. */ worktreeId?: string | null projectRuntime?: ProjectExecutionRuntimeResolution + /** Bypass the host's shared scans because the caller knows disk just changed. + * Optional so an older host simply ignores it and scans as it always did. */ + refresh?: boolean } const ResolvedProjectRuntimeSchema = z.object({ @@ -100,7 +103,8 @@ export const SkillDiscoveryTargetSchema: z.ZodType = z.obj worktreeId: z.string().nullable().optional(), projectRuntime: z .discriminatedUnion('status', [ResolvedProjectRuntimeSchema, RepairProjectRuntimeSchema]) - .optional() + .optional(), + refresh: z.boolean().optional() }) export type SkillFrontmatterSummary = { From 4fdd09af39880810c0f431b9d3ff9e02f4ab9aed Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:27:02 -0700 Subject: [PATCH 02/87] test(terminal): remove fish DECSET handoff races (#14167) * test(terminal): remove fish DECSET handoff races * test(terminal): harden fish query responder From 3ab8b6a11769666b7ac4d43a38a82c771dffaad6 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:36:01 -0700 Subject: [PATCH 03/87] fix(ssh): stop SSH reconnect from multiplying terminals and resuming agents twice (STA-3077) (#13326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ssh): stop reconnect from grafting panes and stacking remote leases Reconnecting an SSH-backed workspace added terminal panes the user never opened, and the remote host accumulated shells nobody was using — one report went from 2 to 19 to 20 relay PTYs across three reconnects (STA-3077). Two root causes, both in the store. Reattach could create UI. `persistPtyBinding` has four creating branches — mint a tab, mint a root leaf, split the root and graft a leaf, mint a layout. All four are load-bearing for `pty:spawn`, which can beat the renderer's debounced layout writer, but none of them is appropriate on reattach, where the pane either already exists or is gone for good. Add `mayCreate`, defaulting true so the spawn path is untouched; every creating branch already sets `terminalMembershipChanged`, so refusing is a check rather than a new code path. Lease identity had no pane key. `upsertSshRemotePtyLease` matched on `(targetId, ptyId)` alone, so a pane that re-leased under a new relay id left its predecessor live with nothing to retire it, and the next reattach fanned out over both. One pane now keeps at most one live lease. Superseded leases are marked `expired` rather than terminated: losing a lease is not proof the shell died, so the remote process is deliberately left running. Tests assert observable behavior rather than mechanism, so they stay valid under any implementation that fixes this. Co-authored-by: Orca * docs(terminal): record the terminal session behavior contract Properties stated as observable behavior rather than mechanism, so an oracle written against them survives a change of implementation. Records the weaker, correct form of the timer rule — a timer may never be the sole cause of a destructive action — because recovery budgets and scratch-file age gates are correct code that an absolute ban would condemn. Also notes which mechanisms are deliberately not required, so each has to earn its place rather than arrive with an architecture. Co-authored-by: Orca * fix(ssh): heal duplicate pane leases that predate pane-keyed supersession Pane-keyed supersession stops new duplicates, but it does nothing for installs that already carry the ones STA-3077 accumulated — the report behind this reached 20 live leases across a handful of panes, and every reconnect fanned out over all of them. Retire the stale duplicates once per reattach pass, keeping the newest lease for each pane under a total order so two hosts resolve a tie the same way. As with supersession, retired leases are marked `expired` rather than terminated: their remote shells are deliberately left running, because a lease we chose not to revive is not evidence the shell died. The relay-session store stubs gain the new method. Note the gap this leaves open: those shells keep running and are no longer reachable from the app, so the "accumulates unused shells" half of the report needs a visible recovery surface rather than a silent kill. Co-authored-by: Orca * fix(terminal): stop respawning a shell that is still running A pane that failed to reattach spawned a fresh shell. Because the restored session id came along, the replacement resumed the same agent session, and two processes appended to one transcript — reported repeatedly, up to five concurrent resumes of a single session. Two defects fed it. The relay reported a source that merely needed re-establishing as `SSH_SESSION_EXPIRED`. The shell was still running; only its output source was gone. Give that outcome its own error so it stops reading as "the session no longer exists". The reattach failure handler then treated every error as proof of death. It checked for expiry and, in the else branch, took the identical action — so the check bought nothing and a transport fault, a timed-out call, or a wedged relay all respawned. Respawn now requires proof: an explicit host expiry or a not-found PTY. Anything else, including an error we have never seen before, is unresolved, leaves the shell running, and keeps the binding for a later reattach. Two existing tests asserted the old behavior. One threw a bare error as scaffolding to reach the spawn-adoption door; it now throws proof, which is what it meant. The other pinned the expiry mapping itself, and now asserts the outcome fails closed *without* being reported as expiry. Co-authored-by: Orca * docs(terminal): record what makes a retention bound safe Shortening a grace period is the wrong lever. Measuring process time and gating reclamation on an independent observation are what make one safe, and they are what deployed systems actually do. Also records that lifecycle belongs in the attach reply rather than a delivered event — that is what removes the need for a durable per-consumer cursor to guarantee an exit is never lost. Co-authored-by: Orca * test(terminal): assert the empty-failure case without an empty Error A thrown empty value exercises the same property — a failure carrying no usable message is not proof the session is gone — and does not trip the empty-error-message lint. Co-authored-by: Orca * fix(ssh): let the durable pane binding outrank recency when retiring leases Choosing the newest lease for a pane is wrong whenever a newer lease exists that no pane is bound to: it retires the lease the pane is actually attached to, detaching a live terminal instead of healing it. Two changes. Arbitration now prefers the lease matching the pane's durable binding, across both the SSH-target and local partitions, falling back to recency only when no binding names either candidate. And supersession at upsert time now defers rather than expiring a bound predecessor. When a lease arrives for a pane that is still bound to a different PTY, the binding has not caught up yet, so both stay live and reattach arbitrates once the binding is available. Co-authored-by: Orca * fix(ssh): roll back a lease retirement whose durable write fails `flush()` logs and swallows write errors, so a failed write left these leases retired in memory while disk still called them attached — and the pane bindings scrubbed alongside them stayed scrubbed. Use `flushOrThrow` and restore both the lease states and the affected session partitions when it throws, reporting nothing retired. Co-authored-by: Orca * test(ssh): prove pane and remote PTY cardinality across reconnects Counts the shells the relay actually hosts, on the container, rather than inferring them from app state — that is the census the report was based on. Asserts the PIDs are unchanged, not merely the count, so a kill-and-respawn cannot pass. Every pane streams before the transport is severed: an idle pane sends no recovery checkpoint, so only a live source comes back needing re-establishment, which is the outcome that used to read as expiry. Co-authored-by: Orca * fix(ssh): actually pass mayCreate:false from the reattach binding write The `mayCreate` guard was correct and had no production caller, so the reattach path still went through the creating branches and grafted panes back. `restoreReattachedPtyRuntime` is that call site — RC3 in the original diagnosis — and it now refuses to create. Binding moves ahead of runtime registration, because registering first would surface a pane the user never opened before the refusal landed. A refusal leaves the remote shell running and reattachable; a *thrown* write stays unknown and still registers, so a failed disk write cannot detach a live pane. Adds an oracle over the call site itself. The store-level tests all passed while the fix was inert, because they called the store directly — only pinning the wiring catches that. Co-authored-by: Orca * fix(terminal): apply the respawn-requires-proof rule to both reattach paths connectPanePty has two near-verbatim reattach blocks — one keyed on the deferred SSH session, one on the restored session — and only the second was fixed. The first still checked for expiry and then respawned unconditionally anyway, so a transport fault there resumed the same agent session a second time. Also keep the wire token out of the pane. The main-process bridge only special-cases expiry, so a source-restore failure crossed IPC as raw `SSH_SOURCE_RESTORE_REQUIRED: ` text and surfaced to the user. It correctly does not respawn; it just should not read like that. Co-authored-by: Orca * test(ssh): state plainly that the reconnect spec is a forward guard It was run against an unfixed tree and passed, so it does not prove the STA-3077 fixes and should not be read as if it does. A clean severed transport does not reproduce the field conditions — accumulated duplicate leases, or a source returning needing re-establishment. It keeps its place as a forward guard: it counts the shells the relay actually hosts and pins their PIDs, so a later change that grafts a pane or respawns a shell fails here. Co-authored-by: Orca * docs(terminal): record that a guard must be pinned at its call site A refusal that exists and is never passed is indistinguishable from no refusal, and store-level tests cannot tell the difference — they call the store directly. Learned from `mayCreate`, which was correct and had no production caller for several commits. Co-authored-by: Orca * fix(ssh): park one PTY's exhausted delivery recovery instead of dropping the channel A per-PTY recovery budget running out disposed the whole relay channel, so one PTY that could not re-prove its delivery aborted every in-flight filesystem and git request on that host and stalled every sibling pane. A retry count is not proof of anything, and it certainly is not proof about the other sessions sharing the channel. Exhaustion now parks that PTY's delivery. The remote shell keeps running, its lease stands, and the next relay open reattaches it with a fresh delivery generation — the parked state is cleared on teardown and the generation changes on reconnect, so a reconnect recovers it. The consecutive-attempt ceiling goes away entirely; the per-generation one is what bounds the retry cost, and the second ceiling only existed to reach the channel drop sooner. Tradeoff worth stating: the failing pane used to self-heal within seconds because the forced reconnect wiped all rejection state, and it now stays frozen until the next relay open. That is a worse outcome for that one pane and a much better one for every other session on the host, and reconnecting is user-reachable. Co-authored-by: Orca * fix(pty): let liveness say unknown instead of forcing it to say dead `IPtyProvider.hasPty` returned a boolean, so a provider whose inventory was empty for reasons that have nothing to do with the session — socket down, cache never hydrated, provider generation just constructed — had no way to say so and answered "absent". Its own siblings already knew better: `probePtyLiveness` and the runtime's `PtyController.hasPty` were both already `boolean | null`, with consumers branching on null correctly. The lie was injected at exactly one interface. Now three-valued, and each provider answers unknown where it cannot prove absence: the daemon adapter off-socket, the SSH provider before a completed listing, the router when any adapter cannot answer, and the degraded provider rather than fabricating a verdict. `terminal_gone` requires unanimous proven absence. Also fixes a real cold-start bug this surfaced: `pty:hasPty` never awaited the daemon-swap startup promise, though the sibling `probePtyLiveness` bridge already did, so before the swap the local provider answered an authoritative false for every daemon-owned id. Net +27 production lines. The plan behind this predicted -92 on the strength of deleting the renderer's dead-session reconcile path; that code is live (`pty-connection.ts` imports it), so nothing was deleted. Expressing a third value where there were two costs lines, and a deletion that is not real is not worth manufacturing. Co-authored-by: Orca * docs(terminal): track the terminal-session correctness handoff package The package was untracked under a gitignored `docs/**`, with the un-ignore rules living only in an uncommitted .gitignore edit — a single `git clean -xdf` would have destroyed the authoritative plan. The 814-path construction snapshot is now pushed as `nwparker/react185-authority-snapshot` too; it had no remote ref. Co-authored-by: Orca * test(ssh): make the reconnect settle window actually wait The settle poll reused a matcher the assertion 15 lines above had already satisfied, and Playwright's poll engine probes immediately and returns as soon as the matcher passes — so it observed the same state twice and elapsed 0ms. A shell grafted a second or two after reattach reported ready slipped through into the next cycle. Reviewer was right on #13111. Test-only; no production change. Co-authored-by: Orca * test(ssh): census both durable session partitions on reconnect Adds a second reconnect scenario and a helper that reads pane records from the local partition as well as the ssh host partition. That split matters: the reattach binding call passes no hostId, so a grafted pane lands in the LOCAL partition and an oracle reading only the host partition passes whether or not the guard is present. Both tests remain forward guards. The second one was reported as discriminating and did not reproduce: with `mayCreate: false` removed from the call site and the app rebuilt, both still passed. Its induction races `pty:kill` against a severed transport, so when the kill lands the lease is cleaned up and there is nothing left to graft. The handoff README is corrected to say so rather than claim a journey. Co-authored-by: Orca * docs(terminal): record the user decision relaxing G6 G6 becomes minimise-and-justify rather than strictly net-negative. The deletion budget the plan assumed does not exist: an entrypoint-rooted import graph found 51 of 53 candidate files reachable and instantiated on live paths, leaving 263 deletable LOC against roughly +1,021 to offset. Correctness may still not be traded for line count. Co-authored-by: Orca * test(terminal): add discriminating oracles for restart, daemon, skew and namespaces Six parallel streams, each required to fail with its guard removed rather than merely pass. Local restart proves the OS process itself survives, by reading `ps -o lstart=` for the shell's own pid. That matters: with the quit path made destructive, the tab, leaf and pty ids all came back byte-identical while the shell underneath was a new process — every existing restart spec would have stayed green. Two separate guards were removed to redden it, and the second reddens only the stale-operation case. Daemon restart discriminates by reverting three-valued `hasPty`; version skew now covers publication semantics and confirms the new `SSH_SOURCE_RESTORE_REQUIRED` token mutates nothing on an old client; two-host isolation censuses both containers. Deletes `src/relay/pty-source-replay-index.ts` — 201 production lines with no importer outside its own test, verified against an entrypoint-rooted import graph rather than a name grep. Five namespace tests are skipped, not passing: they reproduce a defect still live on main where folder-workspace ids compare equal with the instance suffix stripped. PR #12474 fixes it; they are its oracle. Co-authored-by: Orca * test(ssh): induce the reattach graft deterministically instead of racing a kill The previous induction closed a pane while the transport was severed and relied on `pty:kill` FAILING so the lease outlived the pane record. It does not fail: with the provider already torn down, `pty:kill` takes its tombstone branch and marks the lease terminated, and `reattachKnownPtys` filters terminated leases out of the fan-out — so the reconnect never visited the PTY the test was about. It passed on both trees. Seed the precondition instead. Spawn a real remote PTY on a leaf that never becomes a pane, then roll the host partition back to its pre-spawn snapshot, leaving a live lease and a live remote shell that no durable pane owns. No failure races a success. Adds a vacuity guard that is independent of the tree under test: the lease's own `lastAttachedAt` must advance, proving the fan-out actually visited this lease before the pane census is trusted. Verified on this machine under an isolated TMPDIR, since the e2e harness keys its seeded-repo pointer on a machine-global tmpdir path: guard present passes, guard removed fails with the phantom leaf grafted into the local partition, guard restored passes. Co-authored-by: Orca * docs(terminal): propose one authoritative binding identity Every defect this program has touched is the same defect: identity compared with the wrong key, or not compared at all. Lease keyed without the pane, reattach using a creating write, folder-workspace ids compared with the instance suffix stripped, local mutating IPC carrying only an id, a live shell classified as expired, liveness unable to say unknown. Proposal: one branded binding type built from fields that already exist and are already persisted, constructible only from an authoritative source, carried by mutating operations, compared by one shared function. Makes a wrong-key comparison a type error rather than the next incident. Under adversarial review, including against the open issue corpus. Not accepted. Co-authored-by: Orca * fix(pty): refuse mutating operations aimed at a superseded PTY `pty:write`, `pty:writeAccepted` and `pty:resize` accepted any id. The renderer queues input, so a keystroke buffered before a reattach landed on whatever PTY had since taken the pane — and a resize reshaped the successor's shell. Main already tracks `ptyPaneKey` and `paneKeyPtyId` in lock-step, so their disagreement is proof the caller's id was superseded. No wire change, no renderer change, nothing added to the input payload. An id with no recorded pane stays permitted: unowned and orphaned PTYs are unknown, not stale, and unknown never authorizes refusing an explicit operation. That is also what keeps orphan cleanup working — those ids have no pane by construction. The tests pin the CALL SITES, not the predicate. A capability that exists and is never called is indistinguishable from no capability, which is exactly how `mayCreate` sat inert here for several commits with every test green. Co-authored-by: Orca * fix(pty): fence signals at a superseded PTY, and pin why kill is exempt A signal means "interrupt my pane", so delivering one to a PTY the pane has already replaced is a misdirected interrupt. Fence it with the same lock-step proof used for write and resize. `pty:kill` stays deliberately unfenced and a test now pins that: a superseded PTY is orphaned, and reclaiming it is exactly what the orphan-cleanup callers ask for. Refusing there would break the operation that reclaims leaked shells — the opposite of the intent. The fence sits at the IPC boundary, above `tryGetProviderForPty`, so it covers local, daemon and SSH rather than the local path alone. Co-authored-by: Orca * test(terminal): poll the pane binding read so a slower host cannot flake it `readPaneBinding` took a single unpolled read of a DOM dataset attribute immediately after a renderer reload, while its sibling helper polls the same data for 15s. On a native Linux host both tests failed every run with 'No bound terminal pane is mounted' while the app was demonstrably healthy — the screenshot showed the terminal restored with a live prompt and the boot PID echoed. The assertion is unchanged; it is only awaited. Nothing is weakened. Found by running this spec on native Linux rather than assuming macOS behaviour generalises. Co-authored-by: Orca * test(terminal): make the restart identity spec run on Windows too Both probes were POSIX-only and unconditional: `echo ...=\$\$` for the shell's own pid, and `ps -o lstart=` for its start time. Running the spec on a real Windows host proved it dies before reaching either guard, so Journey 1's Windows half was unprovable rather than merely unproven. PowerShell exposes the same two facts as `$PID` and `Get-Process` StartTime. The start time still matters on both platforms for the same reason: a PID alone cannot separate a survivor from a reused number. Still green on macOS. The Windows path is written from the host probe and has not itself been executed end to end — that is the next thing to run there, not a claim being made here. Co-authored-by: Orca * docs(terminal): record the fence's real gap and what peer designs taught Marks the client-constructed binding proposal as rejected with the three false claims that sank it, and records what shipped instead. States the shipped fence's actual limitation rather than leaving it implied: it compares a binding, not an incarnation, so a respawn under a reused ptyId passes. The obvious remedy is wrong here — the agent-create id is deterministic by design so a replayed create stays idempotent, and randomising it would trade this narrow gap for a duplicate-spawn bug. Also records the ranked lessons from four comparable agent IDEs, chiefly that a typed end-reason at end time is what stops a user quit from looking like a resume candidate. Co-authored-by: Orca * docs(terminal): promote Journey 1 to proven on all three platforms The oracle now runs natively on macOS, Linux and Windows, and its discrimination was watched on each: a mutation reddens it, a restore greens it. On Linux and Windows both mutations were run, and the second reddens only the stale-operation test — so the journey's two clauses are proved independently rather than jointly. Windows is the new evidence. The PowerShell branches added blind at ebffb85a848 executed correctly on their first run: `$PID` expanded to real integers, which also proves the pane shell there is PowerShell-family rather than Git Bash, and `Get-Process StartTime` returned kernel start times 5.4s apart — so a recycled pid could not have passed as a survivor. First journey promoted in this program. The other twelve are unchanged, and the residual limit on "every stale exact operation" is recorded rather than glossed. Co-authored-by: Orca * test(terminal): add discriminating oracles for the daemon, skew and multi-host journeys Daemon: replaces a spec that modelled only a client restart and never crossed the daemon boundary, whose successor generation owned nothing so "the live successor is neither killed nor replaced" was vacuous. The PTY leader is now a real login shell reporting `$$` back through the production write path, resolved to a kernel start time. Two mutations each redden exactly one of the three clauses, on macOS and Linux: reverting three-valued `hasPty` reddens only the unknown-not-dead clause; widening the sole-provider fallback reddens only the stale generation clause. Skew: reverting the restore-required publication to expiry reddens 4 of 5 new tests while the legacy control stays green — the regression this branch fixed is now caught if reintroduced. Multi-host: restoring `mux.dispose('connection_lost')` reddens sibling isolation on one host. It does NOT redden across hosts, and that is recorded rather than glossed: a mux belongs to one relay session per target, so its dispose cannot cross a host boundary. Journey 4's cross-host clause rests on isolation-by-construction, not on a mutation. No production code changes. Co-authored-by: Orca * docs(terminal): record journey evidence that falls short of promotion Four journeys now have discriminating oracles but none meets its full stated scope, and each shortfall is named rather than rounded up. Journey 2 is one WSL run from promotion. Journey 12's tests are in-process, so they do not close the live-skew gap the original ledger named. Journey 4's cross-host clause cannot be proven by mutation at all — a mux is per target, so its dispose cannot cross hosts, and the cross-host test stayed green under the mutation that reddens siblings. Journey 13 measured one dimension of ten, on lifted predicates rather than through real IPC. Co-authored-by: Orca * docs(terminal): promote Journey 2 to proven on macOS, Linux and physical WSL The oracle runs on every environment the journey names, and is clause-selective on all three: reverting three-valued `hasPty` reddens only the unknown-not-dead clause, and widening the sole-provider fallback reddens only the stale-generation clause. Selectivity in WSL was established rather than assumed. The spec runs serially, so a red first test reports the others as "did not run" — they were re-run alone under the same mutation and stayed green. Also records that an Orca WSL-mode terminal now starts on that host at all, which it could not before: the distro had no provisioned default Unix user, so every interactive launch blocked on first-run setup. One diagnosis from the WSL run is corrected here rather than repeated: the unrelated `local-pty-shell-ready` failure was attributed to bash 5.3.9, but macOS runs the same bash version and passes 67/67. The trigger is environmental to that distro, and the underlying defect is that the spec pins an absolute count of OSC markers it does not own. Co-authored-by: Orca * docs(terminal): correct the WSL provider-suite diagnosis The WSL run blamed bash 5.3.9 for the unrelated `local-pty-shell-ready` failure. macOS runs the same bash version and passes 67/67, so the version is not the cause — the trigger is environmental to that distro, and the underlying defect is that the spec asserts an absolute count of OSC markers it does not own. Co-authored-by: Orca * test(runtime): unskip the workspace-namespace oracles now their fix has merged These five reproduced a defect that was live on main: folder-workspace ids were compared with the instance suffix stripped, so two workspaces sharing a directory read as the same namespace. They were committed skipped, pointing at the PR that fixes it. That PR is merged, and they pass. Verified they still bite: restoring the suffix-stripping comparison reddens exactly these five and leaves the other four green. An oracle written before its fix, held skipped, and confirmed against the fix after the merge — rather than deleted and rewritten from the answer. Co-authored-by: Orca * test(ssh): add MaxSessions, lazy-discovery and paired-skew oracles Three journeys attempted; none promoted, and the reasons are recorded in the ledger rather than rounded up. MaxSessions=1 against real OpenSSH, with the cap read back from `sshd -T` rather than assumed, and remote pids read on the container two independent ways that must agree, each carrying its kernel start time. Two disjoint mutations discriminate — one reddens only the reconnect clause, the other only the two restart clauses. But the disconnect clause is a forward guard: four separate guard removals left it green, so nothing shipped is load-bearing for it. Lazy discovery samples sshd's own accept log and live session census across a 22s window with the in-use host as a positive control. No mutation reddens its third clause alone — the real cross-host lease scoping is load-bearing, but removing it breaks the sibling host during setup, so the failure carries no clause information. The paired-runtime skew spec pairs two real processes at different versions and refuses to run rather than degrade into a same-version pairing that would look green and prove nothing. No production code changes. Co-authored-by: Orca * docs(terminal): record why the duplicate-resume fix was not built I recommended adding a typed end-reason so a user quit stops looking like a resume candidate, then went to implement it and stopped. `SleepingAgentSessionRecord` already carries three fields that each exist to stop something resuming that should not have — `origin`, `restoreOnTabOpenOnly`, and `automaticResumeBlockedBy` — each traceable to its own incident, consulted at 22 non-test sites. A fourth predicate, however well typed, is the fifth containment cycle. The designs without this bug do not have a better flag; they resume only on an explicit action, into a new terminal id, and make two agents in one terminal unrepresentable in the schema. The first of those is a product decision about whether automatic resume stays a feature, so it is the user's call rather than mine. Co-authored-by: Orca * docs(terminal): reconcile G6 with the recorded decision and assess its clauses G6's body still demanded strictly-negative production LOC after the user relaxed it to minimise-and-justify, so the gate had two conflicting pass conditions and no single truth value. Its body now points at that decision. Assessed the remaining clauses against the branch rather than assuming. Two fail structurally: more than one identity comparison and mutation admission path still exist, and `terminal-input-quarantine.ts` is still reachable from two production files. Records why the quarantine is not subsumed by the superseded-PTY fence, which I had assumed and checked. The fence refuses writes aimed at a stale ptyId; the quarantine guards the user's next keystrokes landing on the successor under its current, correct id — a case the fence never sees. Removing it needs the recovery path to surface a different shell as unresolved, not a deletion. Co-authored-by: Orca * docs(terminal): the input quarantine is load-bearing, not superseded G6 lists "no superseded quarantine remains reachable" and this module was assumed to be one. Disabling its single call site reproduces the hazard it exists for — `cho hi; rm -rf x` reaching the shell — so deleting it without a replacement re-opens command execution. The replacement was costed by building it rather than estimated: +26 production LOC to thread the incarnation, ~+33 complete, and the cross-remount state it needs outlives the destroyed pane so it becomes a module about the size of the one deleted. Floor is roughly +140 to delete 88, and it would add a second identity comparison to a gate already failing for having more than one. The decisive part is that the route is not uniformly available: remote runtime results carry no incarnation, old hosts cannot be made to publish one, and mixed versions are the normal state. A paired client reads unknown, which this program's own rule says is not proof — so either every remote reattach surfaces unresolved, or a fallback is needed and the only correct fallback is this module. Whether to amend the clause or accept something weaker on remote hosts is a user decision, so the clause verdict is left as failing rather than quietly reclassified. Co-authored-by: Orca * refactor(runtime): collapse duplicate identity comparisons G6 requires one identity comparison; five implementations existed across two concepts. Worktree-namespace identity had two: `runtimeWorktreeIdsEqual` and `runtimeWorktreeIdentityKey` independently re-derived repoId plus normalized path. Equality now derives from the key, so the comparison and the sleep / mutation-queue keying cannot drift into two different rules — which is exactly how the suffix-stripping bug reached production once. Pane identity had three byte-identical leaf-UUID comparisons, in orchestration `db.ts`, `lifecycle-reconciliation.ts`, and `orchestration-legacy-process-identity.ts`. One copy moved to `stable-pane-id.ts`, which already owns `PaneKey`, `parsePaneKey` and `makePaneKey` and which all three already imported. No new module, no branded type, no parallel comparison. Net -14 production lines. The namespace oracle still bites: restoring the filesystem parser inside the identity key reddens exactly its five cases. The raw counts are not the actionable set, and the classification is worth recording: of 409 non-test `worktreeId` comparisons, 71 are typeof guards and 81 are sentinel tag checks. Most of the remainder are renderer predicates over store rows where both operands are the same main-minted id, so normalizing there would widen equality rather than correct it. Co-authored-by: Orca * refactor(terminal): finish a half-done fixture move and audit the rest `xterm-bypass-event-fixture.ts` and `__fixtures__/xterm-bypass-event.ts` were byte-identical apart from an import path. The `__fixtures__` copy had zero importers and the live copy compiled as production — someone started the move and left both. Dead copy deleted, live one moved, its three test importers updated. Audited the wider G6 clause by importer rather than filename: 32 test-only files, roughly 3,300 LOC, currently compile as production; 4 of the 36 candidates have real production importers and are correctly placed. The list is recorded in the goalposts. Those 32 are almost all older than this program and outside the terminal surface, so sweeping them belongs in its own change rather than inside a terminal PR. The clause stays failing, with the remaining files named. Co-authored-by: Orca * docs(terminal): the fixture clause already holds where it matters Checked what the build emits rather than reasoning from file paths. None of the 32 test-only fixtures appears in `out/` — Rollup drops them because no production entrypoint reaches them. On "compiles into the shipped product", this clause holds today. On the other reading it cannot be closed by moving files at all: both production tsconfigs use bare `include` globs with no `exclude`, so a `__tests__/` directory matches exactly like any other path, as does every `*.test.ts` in the repo. Relocating 32 fixtures would remove nothing from typecheck scope. A sweep was started and stopped once this was verified, rather than landing 32 moves across areas this program does not own for no gain. If the intent is that typecheck scope should exclude test code, that is a repo-wide tsconfig change with a different owner. Co-authored-by: Orca * docs(terminal): add plain-language design and test overviews Two reviewable documents with diagrams, written so someone with no prior context can follow what breaks, why, and what changed. The design overview explains the five things stacked behind one terminal rectangle, the 2 -> 19 -> 20 report, the three root causes, and the rule underneath all of them: unknown is not dead. The test overview explains why a green test proves nothing on its own, the four-step mutation proof we adopted, and — the part worth reviewing hardest — an honest account of what could not be proven and why, including the properties that are true by construction and therefore have no guard to remove. Co-authored-by: Orca * docs(terminal): add a self-contained visual report of the design and its evidence Pre-renders every diagram to inline SVG in both themes so the report opens offline and stays sharp when zoomed. States the gate/journey score and the retractions alongside the fixes, so the unproven half is as visible as the proven half. Co-authored-by: Orca * docs(terminal): record the finalized two-plane architecture decision Adopts the data-plane proposal and adds the control-plane track it does not cover: re-key ownership by pane, split orphan inventory out, then delete the compensating code. Records that the host-authority alternative was refuted and that the shipped keystroke fence is inert on the reattach path. Co-authored-by: Orca * docs(terminal): add the design brief the review counsel works from Separates verified code facts from unverified leads so reviewers attack the design rather than a reconstruction of it, and records which simpler alternatives were already refuted and why. Co-authored-by: Orca * docs(terminal): report the design counsel's outcome and the live respawn bug it found Three review rounds across two models replaced the two-record split with one leaf-keyed record, deleted attach-time pane identity, and made orphans a connect-time projection. Records that a shipped gesture still turns a healthy remote shell into a duplicate agent resume, and that the renderer classifier in that chain treats an error-message shape as proof of death. Co-authored-by: Orca * docs(terminal): correct the report — the respawn proof gate guards a minority path A final review traced every auto-respawn route. The primary one converts the reattach failure into a boolean before any classifier sees it, so the shipped proof gate never runs there. Records that two of the six shipped changes are narrower than claimed, and why their tests could not have caught it. Co-authored-by: Orca * docs(terminal): explain the landed design on its own terms One leaf-keyed ownership record, orphans computed at connect, and replacement shells only on positive proof — with the shipping order and the one product trade the design asks the owner to accept. Co-authored-by: Orca * docs(terminal): rewrite the design explainer in plain English The first version assumed the reader knew the codebase. Reframed around two bugs, two fixes and one decision, with the jargon replaced by pane / program / note / helper and a five-word glossary for what could not be avoided. Co-authored-by: Orca * fix(ssh): stop reading an identity mismatch as a dead shell The relay reports a pane-identity mismatch by saying the pty was not found, but it found it — comparing identity is how it noticed. Publishing that as expiry made the renderer clear the binding and cold-restore with agent resume, so a live shell gained a second agent on one transcript. Reachable today by detaching a pane into a new tab, which changes the tab the relay froze at spawn. Mismatch now carries its own token and the classifier refuses it as proof. Genuine absence still expires, so a shell that really went away is not stranded. The three failure tokens move to src/shared: main published them and the renderer decided respawn on them, from two copies that had drifted apart. Co-authored-by: Orca * fix(ssh): stop sending pane identity on reattach The relay froze pane identity at spawn, so moving a pane to another tab made it refuse a live shell — and refuse by saying 'not found'. The comparison is presence-guarded, so not sending the fields disarms it on every relay version including ones already installed on hosts: no wire change, no redeploy. Nothing is lost. It existed to catch a relay restart recycling pty-N for a new shell, and in exactly that case pane and tab both still match, so it accepted the wrong shell anyway. The incarnation the attach returns is what distinguishes those, and it already crosses the wire. Removes the whole client-side apparatus: the expected-identity type, its per-lease derivation, its map, and the parameter threaded through four layers. Co-authored-by: Orca * docs(terminal): add tracked goalposts for the new design Each goalpost is a behaviour with an oracle and the mutation that must redden it, so 'proven' cannot be claimed from a green test. Records the anti-inert rule as a first-class goalpost, since three guards in this program passed their tests while sitting off the route production takes. Co-authored-by: Orca * docs(terminal): record that the recovery grant is dead code, deleting a design step The lease stores a relay-native pty id and the caller passes the app form, with a raw equality comparison between them, so the 30s grant cannot fire for a real SSH pane. The death rule that existed to referee it is deleted rather than built, and the dead path itself becomes a removal. Co-authored-by: Orca * docs(terminal): keep the full design detail in the repo It only existed in an ephemeral job directory, so the plain-English explainer had no durable source for its specifics — record shape, death rule, reattach algorithm, migration order and the 25 oracles. Co-authored-by: Orca * docs(terminal): add a resume prompt for a clean session Points at the goalposts as the contract, names the three goalposts whose oracles are already written and red, and carries the process rules that were learned the expensive way — prove guards reachable, verify mutations land, commit per step, and never let a subagent write production files in a shared worktree. Co-authored-by: Orca * test(ssh): add the failing oracles for goalposts S3, S4 and S5 Intentionally RED: 14 clauses that fail against current behaviour and go green under the changes named in new-design-goalposts.md. The branch is held unmerged, so red here means unimplemented, not broken. Each was verified to fail for the right reason and to flip green under the identified fix, which was then reverted. Each pins the producer as well as the consumer, so no clause can pass vacuously if its route is ever severed — the failure mode that let three earlier guards ship inert. Co-authored-by: Orca * fix(ssh): stop fabricating an exit when a reattach fails A failed attach never proves the shell exited. The relay answers not-found for a pane-identity mismatch and for any id it merely cannot hand back, so treating it as death sent the pane a synthetic `pty:exit { code: -1 }`, cleared provider state, deleted ownership and expired the lease — four claims about a process we know nothing about, on a shell that is usually still running. Collapse every failure into the non-destructive branch that already existed a few lines above (`restoreRequired = 'reattachAttemptsExhausted'` + wakeRecovery). A branch collapse, not a new mechanism: goalpost S3. Two tests pinned the deleted premise and are INVERTED rather than patched, so the new intent stays covered: - ssh-relay-orphan-abandon-paths: "retires the lease without a kill when the relay proves the PTY is gone" -> "leaves the shell running when the relay only reports the PTY as not found". Its comment claimed attach verifies liveness before answering not-found; it does not. - ssh-relay-session: "invalidates and broadcasts remote PTYs that cannot reattach" -> "leaves an unreattachable remote PTY alone while its sibling reattaches". Also repairs two clauses left red by c51be8072ba (step A), which dropped the expected-identity parameter and the expectedIdentityByPtyId map. Mutation proof: restoring the destructive block reddens 6 of the 8 oracle clauses in ssh-relay-reattach-exit-proof.test.ts; the 2 producer pins stay green. Verified the mutation landed before believing the result. Net production: -21 lines. Co-authored-by: Orca * fix(ssh): give an SSH pane binding one home An SSH pane's durable binding lived in two persisted partitions. Main's spawn wrote `ssh:`; the relay's reattach write passed no hostId and landed in `local`; the renderer has always published SSH pane membership to `local` on purpose. So `durablyBoundPtyIdForPane` hedged ssh-first-then-local, read a partition no live writer maintained, saw the arriving lease disagree with a stale pty id, and bailed — supersession silently no-opped and both leases stayed live. That is the STA-3077 2 -> 19 -> 20 mechanism. `local` wins: it is the only publisher of pane membership and where `mayCreate:false` is evaluated. Every reader and writer now names it. - resolvePersistedStablePaneOwner / retirePersistedStablePaneOwner drop their connectionId parameter and read the default partition. - the CAS write and both spawn upserts drop the hostId argument (each was an if/else that collapses to one call). - durablyBoundPtyIdForPane stops hedging. - a one-time load fold moves any legacy `ssh:` ptyIdsByLeafId into `local`, preferring `local` on conflict, sequenced after the leaf remap so every folded binding keys on a UUID. Side effect worth naming: the renderer never hydrated the `ssh:*` partition (listKnownRuntimeHostIds filters to `runtime:*`), so the Issue #217 force-quit binding protection had never worked for SSH panes. It does now. Mutation proof, run as a 2x2 because the two edits can mask each other: - fold disabled, reader local-only -> 1 clause reddens (the fold is live) - fold disabled, hedge restored -> 3 more redden (the reader is live) - fold enabled, hedge restored -> ALL GREEN That last row is why this commit adds an eighth clause: with the fold shipping, the fold erases the divergent copy at boot, so the reader guard would have shipped unproven — the exact failure mode G5 exists to catch. The new clause rewrites `ssh:` mid-session (orphan adoption still writes there) and reddens when the hedge is restored, pinning the reader on its own. Five clauses in ipc/pty.test.ts pinned the two-partition shape and are INVERTED to the single home, each keeping an explicit arity check so a re-added partition argument fails loudly rather than silently. Net production: +9 lines (the fold is new state repair; the call sites shrank). Co-authored-by: Orca * fix(ssh): bind a pane through one producer so the fence is live on reattach The superseded-PTY fence refuses a keystroke queued for a shell whose pane has since bound a different one. It reads `ptyPaneKey` disagreeing with `paneKeyPtyId` — and only spawn ever wrote those maps. Reattach bound the pane through `runtime.registerPty` instead, so the maps never learned the successor, `isSupersededPtyId` returned false by construction, and the fence was inert on the one path it was built for. Goalpost S5; a defect in already-shipped work. Collapse to one `bindPaneShell` producer that writes the durable record and the fence maps together. All three binding paths call it: the relay reattach and both spawn handlers. Error policy stays at the call sites because it genuinely differs — a caller that just created a shell must clean it up on a failed durable write, a caller that merely reattached must not detach anything. The paneKey is composed from the tab that holds the leaf *now*, resolved from the live layout, not from the tabId frozen in the lease. Only the leaf half of a pane key is remint-stable; `detachTerminalPaneToTab` moves a live pane and its PTY into a new tab, so a stored tabId names the tab the pane left. Mutation proof, both sub-guards isolated: - drop the `rememberPaneKeyForPty` call -> 3 clauses redden - prefer `args.tabId` over the live layout -> 1 clause reddens The second clause is new in this commit. Every pre-existing clause in the fence oracle used one tabId on both sides, so a producer that simply forwarded `lease.tabId` would have gone green and shipped the tab bug unnoticed. Two source-text clauses are STRENGTHENED, not relaxed. They previously required the relay to hold a `persistPtyBinding` call of its own and merely forbade an ssh-partition argument on it. The relay now has none, so they assert ZERO direct binding writes there plus a `bindPaneShell` call — a second bind producer is exactly the defect this removes. Also repairs a latent false green: the "persistence fails" case in ssh-relay-session-reconnect-incarnation was passing because a missing mock made the call throw a TypeError that happened to emit the console.error it asserted. The failure is now injected at the producer, so it is a real oracle for "a thrown durable write must not detach the PTY". Net production: +60 lines. This is the one step in the program that grows; the shrink arrives with S8's deletion. Reported rather than smoothed over. Co-authored-by: Orca * feat(terminal): show an unreachable pane as disconnected with two actions Ships with S3. Collapsing the fabricated exit removed a lie, but it left the pane frozen: `restoreRequired` never crosses to the renderer, so a relay-driven reattach failure had no user-visible signal at all, and the renderer's own reattach arms showed a raw error toast with no way to act. An unproven failure now renders the pane as disconnected with exactly two explicit actions — "Try again" (remount against the same shell via requestTerminalPaneRecovery) and "Start a new terminal" (retire the binding, then spawn fresh). Nothing infers death and nothing auto-spawns; the user decides, because at that point no one knows whether the shell is alive. The two actions are the same two things the code already did, moved behind a click: the retry is the existing pane-recovery request, and "start a new terminal" is the existing clearExitedPanePtyLayoutBinding + clearTabPtyId + startFreshColdRestoreAgentResume sequence that used to run automatically on a "proven gone" error. No new IPC channel: the silent-respawn decision was always renderer-local. Copy constraint, enforced by an oracle rather than a review note: the banner may never assert the shell exited. STYLEGUIDE.md:236 already forbids result verbs without result data, and a failed attach is not result data. A test asserts the rendered text matches no death verb and shows no wire token. `TerminalRemoteRuntimeReconnectBanner` is renamed `TerminalPaneDisconnectedBanner` — it now serves any transport, and per AGENTS.md the name must say what it holds. Existing i18n key strings are kept verbatim so no shipped translation breaks; the SSH copy is additive (4 new en.json keys). `describeReattachFailure` is deleted with its last caller. Its two cases were not dropped: "keeps the wire token out of the pane" is re-asserted against the new copy, which is a stronger place for it. Renderer production (excluding the pure rename): +28 lines. * refactor(terminal): delete the SSH pane recovery grant, which could not fire For a disconnected pane, `recoverTerminalPane` consulted a recently-expired SSH lease and, on a match, spawned a replacement shell. It could never match: leases store a relay-native pty id (`pty-7`, normalized on every write) while the runtime registers the app-form id (`ssh:@@pty-7`), and the comparison was a raw `===`. The branch was also unreachable for a local pane, which has no SSH lease. Goalposts S6 and S8. A characterisation oracle lands FIRST and proves it, rather than assuming it: ssh-pane-recovery-grant-reachability.test.ts mints BOTH id forms from the production helpers — never as two hand-typed literals — so it tracks the real namespace split instead of restating it, and seeds a lease that qualifies on every other predicate (state, worktree, tab, leaf, grace window). Anti-vacuity assertions pin that control actually reaches the gate rather than bailing early. Mutation proof, run before the deletion: normalizing the comparison at the `lease.ptyId === ptyId` site makes the grant fire and reddens the oracle. That is the exact "fix" someone would reach for, so the oracle is pinned to unreachability rather than to the throw. Deleted: the grant tail, the `terminalPaneRecoveryByIdentity` dedup map (whose only consumer was the grant), and the dead `ptyId` parameter. NOT deleted, and worth naming because over-deleting here would break users: `getRecentExpiredSshLease` itself, `hasRecentExpiredSshLeasePane` and `SSH_PANE_RECOVERY_GRACE_MS` all stay. Their other two callers pass `ptyId` undefined, which short-circuits the broken comparison — those are live today and feed headless-mobile terminal-tab visibility. Also NOT done: deleting only the gate while keeping the spawn. That would have granted a respawn to every disconnected pane — a behaviour change in the dangerous direction. The refusal is what stays. Four tests pinned the grant. All were seeded through a helper that stores the lease id as the same literal it registers as the runtime pty, with a null connectionId — a shape production cannot mint. Three are INVERTED, keeping their scenarios; the fourth is now tautological and carries a comment saying so rather than being left silently hollow. The oracle's own counterfactual control is inverted by this deletion too, which is recorded in the file: its flip from grant to refusal is what "inert" means here. Honest limit, stated in the oracle rather than smoothed over: unreachable BY CONSTRUCTION for SSH panes; for a local pane, unreachable only up to a random UUID collision. Net production: -32 lines. * docs(terminal): record S3, S4, S5 and S8 proven, and G1 missed All seven step goalposts are now proven. G1 (net-negative production) is NOT met at +83 and is reported as a miss with a per-step breakdown rather than reframed. Also records the near-miss G5 caught: S4's reader guard showed no mutation response because the load fold had already erased the divergent state at boot. It was correct and would have shipped unproven. An added clause pins it. * fix(terminal): a reattach not-found is not proof the shell is gone Closes the last live route to the reported duplicate agent resume (RC2), found by the E2E harness rather than by reading: when a relay is stalled and replaced, the fresh relay has no memory of `pty-1` while the old shells keep running under its predecessor. It answers not-found, and the renderer read that as proof. A not-found means the relay WE ASKED cannot hand that id back. That proves an exit only if the relay process that minted the pty is the one answering. The design says exactly this (D3 row 2), and gates the grant on `relayInstanceId` equality — a field step E-2 never built. `SSH_SESSION_EXPIRED` is not independent evidence either: its ONLY producer is that same not-found mapping in reattachSshPtySession, and the token's own doc comment claimed "the host proved the session is gone", which it never did. So `isProvenSshSessionGoneError` returns false. Both reattach arms now always take the non-proof path and surface the pane as disconnected, which is what the owner approved in D1 — and what makes that affordance load-bearing rather than near-unreachable, since it was previously only reached by errors that were already rare. The respawn tails are deliberately NOT deleted. The design preserves the grant as a conditional for E-2, so the decision point stays and a clause pins that nothing reaches it meanwhile. This is the one place in the program where an unreachable branch is kept on purpose, and it is labelled as such. Tests: three clauses asserted a not-found proves death and are INVERTED, with the reasoning recorded. The #12101 cold-restore case reached the spawn door by throwing a not-found; that door is now opened by the user's "Start a new terminal", so the test drives that instead and keeps all four of its original assertions verbatim — strictly better coverage, since it now pins that the automatic respawn stopped AND that the door still works. Mutation proof: restoring the old predicate makes the new no-respawn clause fail with "expected connect to be called 1 times, but got 2", confirming it reddens on the real production route rather than passing vacuously. * test(ssh): add E2E oracles for pane cardinality and duplicate resume Both reported failures now have end-to-end coverage against a real Docker OpenSSH relay, gated on ORCA_E2E_SSH_DOCKER like the rest of the suite. - ssh-reconnect-pane-cardinality-across-partitions: three real reconnect cycles; after each, pane ids unchanged, exactly one live lease per leaf, exactly one remote shell per pane key, and exactly one pty id bound per leaf ACROSS BOTH durable partitions. Two partitions agreeing is tolerated; two naming different shells is the S4 divergence and fails. PASSES. - ssh-reattach-does-not-resume-agent-twice: the host itself records one line per shell launch via a .bashrc hook keyed by ORCA_PANE_KEY, so a duplicate resume is counted at the source rather than inferred. Fault is SIGSTOP on the detached relay. PASSES. - ssh-disconnected-pane-affordance: written whole but held as test.fixme. The banner needs the target CONNECTED while a single pane's attach fails, and both host-side faults drove the target out of connected instead. Held rather than deleted so it runs the day a seam exists; the reason is measured, not assumed. New helpers: docker-ssh-relay-stall (SIGSTOP/SIGCONT, reads the stop back off /proc so a fault that did not land cannot make an oracle vacuous), and remote-pane-launch-transcript. Note for whoever picks these up: a stalled relay leaves two detached relay groups on the host, and readDockerSshRelayProcessSnapshot throws on more than one, so call it before the fault. Writing these is what surfaced RC2 surviving S3 — fixed in 7cd7fef927f. * fix(ssh): fold the pane incarnation with the binding it fences Found by adversarial review. `persistPtyBinding` writes the binding and its incarnation into the SAME partition, and the incarnation is what its CAS compares. Step P's load fold moved only `ptyIdsByLeafId`, so after upgrade a pane whose incarnation had been written to `ssh:` kept the guard in the partition nothing reads: `resolvePersistedStablePaneOwner` read `undefined` from `local`, the CAS then compared undefined against undefined, and the incarnation half of the fence passed for any value until the next write healed it. Not data loss and not a wrong-shell bind — the ptyId half still held — but a guard silently weakened by a migration is the exact shape this program keeps finding, so it is closed rather than noted. Mutation proof: disabling the incarnation half of the fold reddens the new clause; the other eight stay green. One clause in persistence.test.ts asserted the incarnation survives a reload in the SSH partition. Its subject is that the reconciled value is preserved, not which partition holds it, so the assertion follows the binding to its one home and additionally pins that the ssh partition no longer keeps a copy. * docs(terminal): record the two defects found after the goalposts were met RC2 survived S3 and was found by writing the E2E test, not by reading. The load fold left the incarnation half of the pane fence behind and was found by adversarial review. Both were invisible to unit oracles that had already gone green, which is the useful part of the record. * fix(terminal): close four defects found by adversarial review Four reviewers over the diff, every finding put to an independent skeptic. Nine of twelve agents died on prompt length, so most findings arrived UNREFUTED rather than refuted — I checked those myself instead of counting them clean. Four were real. 1. "Start a new terminal" resumed the agent instead of starting a new one. The action passed the cold-restore startup, which carries the agent's providerSession. That was correct for the automatic respawn it replaced, because that only ran on PROOF the shell was gone. Behind this button the shell is probably still alive, so it put a second agent process on one transcript — the exact defect this pane exists to prevent, reintroduced by the fix for it. Now starts a genuinely fresh shell, which is also what the button says. 2. The banner was never retracted. The app-SSH transport publishes no recovery states, so nothing cleared the card: it sat over a live shell with armed buttons. Both actions now clear it before acting. 3. The load fold destroyed bindings it could not move. When a tab existed only in the ssh partition there was no local layout to fold into, and the code cleared the source anyway — deleting the only record of that binding. It now leaves such a tab alone: with no second home there is nothing to disagree with and nothing to fold. 4. The relay registered the pty under the lease's frozen tabId while bindPaneShell bound and fenced it under the live one, splitting a moved pane across two tabs and ensuring a mobile surface for the tab it left. bindPaneShell now returns the resolved tabId so both use one coordinate. Also repairs a reliability-gate manifest entry the banner rename broke. That would have been caught by the pre-commit lint gate, which I had been skipping with --no-verify; the full-repo sweep caught it instead. Mutation proofs, each verified to land before being believed: - restoring the resume reddens on `registerAgentLaunchConfig` and, with that clause disabled, on the spawned command being "codex '--dangerously-bypass-approvals-and-sandbox' 'resume' 'codex-session-1'" - disabling the fold's move reddens the ssh-only-tab clause - registering under lease.tabId reddens the moved-tab clause The first clause was VACUOUS on its first attempt and is recorded as such: the fixture had no resumable agent, and it asserted a field name the spawn path does not use. It now pins the fixture itself, so it fails loudly rather than going quiet again if `codex` stops being resumable. * fix(terminal): drop the remote-host assumption from the disconnected copy The banner said the shell 'may still be running on the host'. The reattach arm that publishes it is not SSH-only — a local or folder-workspace pane reaches it too, and there is no host to speak of there. The claim that matters is that the shell may still be running, which holds either way. * fix(terminal): close round-2 review findings, including one regression Round 2 ran narrow per-area scopes so agents stopped dying on context: 7 of 7 reported, versus 3 of 12 in round 1. Two findings survived refutation and both were real. 1. REGRESSION I INTRODUCED. `bindPaneShell` resolved the tab from the live layout for EVERY caller. That is right on reattach, where the lease's tabId is the frozen side — but backwards on spawn, where the caller's tabId is fresh truth and the persisted layout is the stale side inside the renderer's publish debounce. Breaking a pane out into a new tab and spawning into it in that window resolved back to the tab the pane had just left, writing the durable binding and the fence under one tab while the lease and the runtime registration used the other — the split-coordinate defect step F exists to remove, reintroduced on the spawn path. Live-layout resolution is now opt-in via `tabIdMayBeStale`, set only by the reattach bind. A clause pins that no spawn-side call sets it. 2. The fold moved every incarnation even for a binding it had deliberately left in place, splitting a pane's binding from the incarnation that fences it — the same defect 994733d8b1a closed, in the other direction. Incarnations now move only with the binding they belong to. The reload clause that had been inverted for the fold is restored to its original assertion, because an ssh-only pane now correctly keeps both halves together. Also closes a live route to the reported duplicate resume that my earlier fix missed. `isProvenSshSessionGoneError` covered the rejected-promise arms, but a reattach can also report expiry through the transport's error callback and then resolve falsy; those two branches still cleared ownership and cold-restore resumed the agent. A skeptic refuted this as pre-existing rather than caused by this branch, and that is correct on causation — but it is a live second route to the exact defect this work exists to remove, so leaving it would make the claim that duplicate resume is fixed false. Both branches now surface the disconnected pane. Five tests pinned that callback respawn. Two are INVERTED to the disconnected outcome; two keep their real subject (stale-callback fencing, delayed parked snapshot) and now reach a replacement shell through the banner's "Start a new terminal", which is the new production route; one — the cold-restore resume after expiry — is inverted to assert no resume command and no agent launch config, with a fixture pin so "no resume" cannot pass vacuously. * fix(terminal): close round-3 review findings on tab resolution and the fold Round 3, narrow scopes again: 9 of 9 agents reported. Two findings confirmed. 1. A thrown durable write lost the resolved live tab. `bindPaneShell` resolved the tab internally, so when `persistPtyBinding` threw, the relay's `bind` stayed null and it registered the pane in the runtime graph under the frozen lease tab — splitting the graph from the durable record it had just moved. Resolution is now an explicit `resolvePaneShellTabId` the relay calls BEFORE the write, so a throw cannot lose the answer. This also deletes the `tabIdMayBeStale` flag added a commit ago: the reattach resolves its own tab and passes a live one, and spawn callers simply pass theirs. The distinction is now carried by which caller resolves, not by a flag they must remember. 2. The fold could pair a binding with an incarnation that was never written alongside it. Where both partitions named a leaf, local won the binding but the incarnation was copied across independently — so local's pty could end up fenced by the superseded partition's incarnation. `persistPtyBinding`'s CAS compares both, so that pane's next legitimate update would be refused. An incarnation now moves only when the pty it belongs to is the one that ends up bound. My first attempt at this over-corrected and skipped the case where both partitions name the SAME pty — where the incarnation does belong with it. The existing clause caught that immediately, which is the fixture doing its job. Also hardens `findTerminalTabIdForLeaf`: it now requires the tab to still exist in `tabsByWorktree`. A layout entry outlives the tab it described, and binding a live shell to a deleted tab registers a pane under a ghost and can resurface it. The fence oracle's fixture gained the live tab it was missing, so that clause cannot pass by resolving nothing. New clause covers the divergent-pty case the reviewer named — the two partitions naming DIFFERENT ptys for one leaf, which is the divergence being migrated and was previously untested. * fix(terminal): check tab liveness without depending on a worktreeId match A reviewer asked, correctly, whether lease.worktreeId always matches the tabsByWorktree key exactly — including for a folder workspace, whose worktreeId carries a `::workspace:` suffix and is matched by full-string equality. Rather than assert that invariant, this removes the dependency on it. Tab liveness is now checked across every worktree instead of under one key. A leaf id is a UUID, so there is nothing to disambiguate by worktree, and the resolver no longer has an answer that depends on two strings agreeing — which is exactly the class of full-string comparison that produced issue #12474 in this area. Had they diverged, resolution would have silently returned undefined and fallen back to the stale lease tab, quietly restoring the moved-pane bug for folder workspaces only. Failing open like that is worse than the check itself. * style(terminal): keep the membership authority under the max-lines limit My previous comment pushed the file to 301 lines. AGENTS.md forbids a max-lines disable or a per-file bump, so the comment is trimmed to the repo's concise standard and the liveness check folded into the existing condition. * fix(ssh): resolve every incarnation in the pass that clears its binding Round 4 confirmed one defect, in my own round-3 fix. The filter that stopped an incarnation following a LOSING binding also stopped it being deleted — while the binding itself was still cleared. So a conflicted leaf left the superseded incarnation behind with nothing to fence: durable fence state in a partition holding no binding, which is the one-home invariant this step exists to establish, broken by the code establishing it. The reviewer also named the test gap exactly: the clause I added asserted only that the value was not copied into local, never that it was gone from the partition it lost in. Both are asserted now. Incarnations are resolved in the same loop that clears the bindings, so no binding can be cleared without its fence being resolved. Three outcomes, by which pty ends up bound: - this binding moves (local had none) -> its incarnation moves and OVERWRITES any local value, because a local incarnation with no local binding is a leftover rather than a fence. That case previously synthesized a pair. - both name the same pty -> keep whichever fence local holds. - local wins with a different pty -> drop this incarnation with the binding it belonged to. The second and third outcomes were flagged by the same reviewer as real but attributable to my earlier commit rather than that one. They are the same defect class, so they are fixed here rather than filed. Two new clauses: the superseded incarnation is deleted, not merely uncopied; and a moving binding overwrites an orphaned local incarnation. * fix(ssh): never pair a moving binding with a fence that was not written for it Round 5 ran a mechanical ten-case matrix over the fold twice, independently. Both passes landed on the same primary defect, and it is one my previous fix created. Case 7: the ssh partition holds a binding with NO incarnation, and local holds a stale incarnation with no binding. The binding moves into local and inherits that leftover, producing (arriving pty, unrelated incarnation) — a pair no writer ever produced. persistPtyBinding's CAS compares both halves, so the pane's next legitimate update is refused. The previous fix only replaced local's leftover when the ssh side had an incarnation to replace it WITH; absent one, the leftover survived. A moving binding now takes the ssh fence whatever it is, including absent, in which case local's is deleted. Also fixes the bookkeeping both passes flagged: the function defaulted `workspaceSession` at the top, so a profile with no local session could be mutated on a path that then returns false — a mutation with no save scheduled. It now returns early instead: with no local session there is nothing to fold into, which is also the honest reading. Two findings are deliberately NOT fixed, recorded rather than silently dropped: - An ssh incarnation whose binding was already missing BEFORE the fold survives, because iteration is binding-driven. It is a pre-existing orphan isolated in a partition no reader consults for pane bindings, and reinterpreting it is not this migration's business. The comment claiming an incarnation never outlives its binding overclaimed and is corrected to say what the code does. - Two ssh partitions carrying the SAME tab and leaf resolve by object-key order. A tab belongs to one worktree on one host, so this is not a shape production writes; making it deterministic would mean inventing a precedence rule for a state that should not exist. New clause covers case 7 directly. The matrix cases both reviewers named as uncovered are now covered except the two above. * refactor(ssh): delete the pane-binding fold; its premise was false The migration moved legacy `ssh:` pane bindings into `local` and cleared them, on the theory that the ssh partition was a stale spill of the desktop plane's state. Investigation of both hypotheses the team raised disproved that: - NOT cross-version compat. The introducing commit says it is for CONCURRENT multi-host. Nothing about the partition crosses the wire (zero references under src/main/runtime/rpc/), and the payload that does cross the SSH boundary — RemoteWorkspaceSnapshot — is projected from `local`. The only downgrade-compat comment protects `local`, the other direction. - NOT multi-client. PersistedState is one file on one machine; phones and CLI are RPC clients into that same process. Orca's real per-client state is `mobileClientTabSelectionsByDeviceId`, 13 lines above in the same struct, and it carries selections only — never a ptyId. What it actually is: the headless/CLI/mobile plane's OWN home, written and read deliberately across several tickets (STA-3463, STA-3465), with tests that assert that partition by name. So the fold was not tidying a spill — it was erasing another plane's live state. Measured symptom: a split SSH tab would disappear from mobile while its shell kept running. The desktop plane's fix never needed it. Supersession multiplied panes because `durablyBoundPtyIdForPane` hedged into the other plane's copy; reading `local` alone is the fix, and it stands without any migration. Deleting rather than redesigning, because the redesign had no target: five review rounds each found a defect in that function, three of them inside the previous round's fix, and every one was a cell of a merge matrix that only existed to serve a premise that was false. The `boundPtyIdsAcrossPartitions` clause is INVERTED, not dropped. It required the two partitions to AGREE after load — which encoded the false premise, and is what made a migration look necessary. It now asserts the narrower, stronger property: what the desktop plane resolves follows `local` alone, whatever the other plane holds. Also proven, and the reason unification was NOT attempted: worktreeId is `::` where repoId is a randomUUID minted client-side at repo add (orca-runtime.ts:18722), so two servers cannot collide on one. Host scoping was not protecting against that here — but the two planes' opposite choices are each deliberate and each test-pinned, so choosing a winner is an architecture call, not a cleanup. Net production: -75 lines. * fix(ssh): arbitrate on the pane, not the tab its lease was written in Correctness review round 1 on #13326: 5 raised, 2 confirmed, both real. 1. Arbitration looked the durable binding up under the lease's FROZEN tabId. `detachTerminalPaneToTab` moves a live pane and its PTY, and nothing re-keys a lease, so after a break-out the binding lives under the new tab and the lookup found nothing. The bound shell then lost to recency and supersession expired the pane's OWN lease; the follow-on scrub could not clean up either, because it matches lease.tabId against the layout's tabId. Reconnect skipped the expired-but-bound PTY and reattached the stale winner onto the pane. The leaf is the stable half of pane identity, so the lookup now prefers the named tab and falls back to wherever the leaf actually is. 2. Arbitration read the desktop plane only while the scrub reached into `ssh:`, so a headless-plane pane was invisible to the ranking and then lost its live binding to it. Fixed at the reader, not the scrub: local FIRST, headless plane only as a fallback. That is NOT the STA-3077 hedge — that bug was preferring `ssh:`, letting a copy no live writer maintains outvote the real binding. A fallback consulted only when local is silent gives a headless-owned pane a vote without ever outranking a live desktop binding. Proven by mutation: swapping the two back to ssh-first reddens 4 clauses, including the ten-reconnect cardinality one. Also closes two fulfilled-result routes to the duplicate agent resume that the earlier RC2 fix missed — `handleReattachResult` respawned on a result flagged `sessionExpired` and on a result carrying no pty id. A skeptic refuted both as pre-existing rather than PR-caused, which is correct on causation, but they are live routes to the defect this PR claims to fix. Scoped to SSH panes only. The first attempt diverted every pane and broke three daemon tests — correctly: a local provider is authoritative about its own ptys, so "cannot reattach" there is not the ambiguous evidence it is for a relay that may have been replaced. New clause covers both routes; disabling the diversion reddens it. 43,253 pass; the 7 failures are pre-existing on clean main in this environment. * fix(ssh): supersede on the leaf, so a moved pane retires its own predecessor Correctness round 2. This is the reported cardinality growth in its surviving form, and it is the sharpest finding of the review so far. Supersession matched sibling leases on `(worktreeId, tabId, leafId)` and bucketed duplicates under the same key. A lease freezes its tabId when written, and `detachTerminalPaneToTab` moves a live pane — so after a break-out the pane's next lease carries the NEW tab and its predecessor carries the old one. The two never match, the predecessor is never superseded, and the live count grows on every reconnect. Exactly the reported 2 -> 19 -> 20, for any pane that has been moved between tabs. Round 1 fixed the same frozen-tabId mistake at the binding LOOKUP. It did not fix it here, at the sibling match and the bucket key, which is why the bug survived a round. Both are now keyed on the leaf — the stable half of pane identity, per stable-pane-id.ts: the tab half changes on break-out. Two clauses cover it: a predecessor whose lease names the tab the pane left is superseded, and the live count stays flat across ten reconnects that each land in a new tab. Restoring `tabId` to either the match or the key reddens both. E2E re-run against a real Docker relay after the change: 2 passed. Full suite 43,255 pass; the 7 failures are pre-existing on clean main in this environment. * fix(ssh): a supersession decided from one plane only mutates that plane Correctness round 2 confirmed finding. Arbitration ranks leases using the desktop plane's binding, then handed its losers to a scrub that walked BOTH planes — so the headless/CLI/mobile plane's binding was deleted for a lease it never got to vote on. Its owner then has no durable record to reattach that shell by, and can fall back to recency or orphan a shell that is still running. `clearSshRemotePtyBindingsForLeases` now takes `arbitratedFrom`. A decision reached by reading one plane may only mutate that plane. An explicit expiry or termination is plane-agnostic — the pty is gone for everyone — so those callers pass nothing and still scrub both, which is what the existing `markSshRemotePtyLease` oracle pins. I initially assessed this as not-a-defect, reasoning that local winning IS the STA-3077 fix. That was about RANKING and did not justify DELETING the other plane's record; the round-2 verdict was right and I was wrong. It did also catch a comment of mine that had gone stale — a clause claimed the other plane was "deliberately left alone" while the code cleared it. The comment now says what the code does and why. Mutation: letting arbitration scrub both planes again reddens the new clause. 43,256 pass; the 7 failures are pre-existing on clean main in this environment. * fix(terminal): put the unreachable-pane guards in one place Correctness round 3. The leaf-keying from round 2 came back clean, twice and independently. But the renderer produced findings for a third consecutive round, and they were all one shape: `publishUnreachablePane` is called from seven sites, each needing the same guards, and a different one was missing at each. So this stops patching sites and moves the guards into the publisher: - `disposed` — a late rejection republished a card for a numeric pane id that had already been reused, giving a fresh pane a phantom card whose actions closed over a dead session. - `connectionId` — the deferred catch is not SSH-only. A local/daemon pane could be shown an SSH ambiguity card it can never clear, and would loop: retry remounts, reattach rejects, card returns. Its provider is authoritative about its own ptys, which is exactly why the two branches in handleReattachResult already had this guard — and why the catch needed it too. Two more real defects in the banner's own actions: - "Start a new terminal" passed `null` to suppress the saved agent startup, but connect FALLS BACK to the startup its transport was constructed with whenever a per-call field is absent (pty-transport.ts). So the "fresh" shell could resume the same provider session — the duplicate transcript this pane exists to prevent, for the third time in this button. `suppressSavedStartup` makes the suppression explicit; `??` means passing null could never have worked. - It also cleared both durable bindings BEFORE the spawn, and discarded the promise. A spawn that resolves null left the pane blank, unbound, and with no way back to a shell that may still be running. Bindings are now cleared only once a replacement actually starts, and the card returns if it does not. - "Try again" voided its boolean; a declined remount left no card and no shell, strictly worse than the toast it replaced. It now republishes. The strengthened clause is the point: the old one asserted the ABSENCE of per-call startup fields on a mocked transport, which passes whether or not the production fallback fires. It now asserts the explicit suppression, and reddens when the flag is dropped. Four fixtures were under-specified — they drive SSH reattach scenarios but never seeded an SSH repo, so `connectionId` was null and they had been passing without the pane being SSH at all. Seeded, not weakened. 43,256 pass; the 7 failures are pre-existing on clean main in this environment. * fix(terminal): suppress the whole saved startup set, not field by field Round 4 self-check on my own round-3 fix. `suppressSavedStartup` guarded four of the six values `connect` falls back to — `launchAgent` and `startupCommandDelivery` still inherited from the transport's constructor. Adding the guard per field is precisely how those two were missed, and the guarded expressions had become unreadable. The saved values are now one object that `suppressSavedStartup` drops wholesale. A field added later is covered by construction rather than by remembering. Found by asking the question the review lens was given rather than waiting for its answer: does the suppression cover EVERY channel, or only the ones I noticed? It did not. * fix(terminal): stop stranding a local pane whose restore fails The unreachable-pane card is SSH-only: a local, daemon or runtime-host pane has no connection to be unreachable ON. Consolidating that guard into the publisher made it silent, and four callers paired the now-conditional publish with an unconditional `return` — so on the deferred-reattach path, which unlike the direct-SSH one is not nested under `connectionId`, a local pane got no card, no error and no replacement. Frozen, with a stale binding. The diversion now reports whether it took ownership of the failure, so a caller can only stop when something actually handled it. A pane that cannot show the card falls through to the replace-in-place it had before, which is correct: its provider is authoritative about its own ptys. The two direct-SSH sites are inside `if (connectionId)`, so they are unchanged in behaviour; the return value simply makes the pairing impossible to get wrong at the next call site. * docs(terminal): put two comments back on the thing they describe The lease-healing docblock had drifted onto durablyBoundPtyIdForPane, which neither retires leases nor returns a count, leaving the real healing entry point undocumented and its neighbour carrying two contradictory descriptions. The suppression comment claimed to cover "every startup value this transport was constructed with"; env/envToDelete are constructor values and deliberately stay outside the set. Nothing behavioural changes here — but a comment that overstates a boundary is how the next maintainer picks the wrong one. * docs(terminal): do not claim a remote runtime is authoritative about its ptys A remote runtime reaches its pty over a connection that can fail, so a failed reattach proves no more there than it does over SSH. It keeps replacing in place only because the card is SSH-scoped, not because its provider is authoritative. Say that, so the limitation is deliberate rather than implied. * perf(ssh): stop fsyncing the whole store from the main thread on reconnect supersedeDuplicatePaneLeases runs at the top of every reattach pass, and when it retires anything it flushed synchronously. flushOrThrow fsyncs a multi-MB file from the Electron main thread — this file already documents (see flushAsync) that on a stalled network profile mount that syscall is uninterruptible, so the app stops repainting and no deadline can bound it, because the deadline's own timer is stuck behind the same block. The population that hits this is exactly the one the heal exists for: an upgraded install carrying accumulated duplicates. It now awaits the async durable twin its neighbours on this path already use. Still "OrThrow", because a retirement that is not durable must not be believed — the rollback is unchanged. * fix(ssh): make a parked PTY delivery expire instead of going dark for good Exhausting the per-generation recovery budget parks one PTY's delivery rather than dropping the shared relay channel — right, because the channel is shared and a retry count proves nothing. But the only escape it named was "the next relay open", and a channel that stays healthy never gives it one. That leaves a pane with no output and no way back, which is the exact state this change exists to prevent; before this branch, exhaustion dropped the channel and the reconnect ladder recovered the pane (loudly, at every sibling's expense). The park is now a cooldown rather than a verdict: the next rejected frame after it starts a fresh budget. Recovery is rate-limited, never abandoned, and the containment that made parking right in the first place is untouched. Elapsed time, not a timer, so there is nothing to cancel on teardown and no late fire after dispose. * fix(ssh): let a due park past the retired-delivery filter, and prove it The cooldown added in 124e00e8a83 was reactive: it needed a later rejected frame to reach recovery. But retirement is ours, not the host's — a stalled source keeps publishing the very token we retired, and acceptPtyData drops those frames before classification. So the wake-up could never fire in the case that actually happens, and the pane stayed dark exactly as before. A parked PTY past its cooldown is now let through that filter. It costs nothing: the frame is re-classified as rejected and re-retired, so no output reaches the terminal — it only regains the ability to ask for recovery. The test shipped with that commit could not have caught this: it woke recovery with a fabricated NEW delivery token. Retirement holds one key per relay PTY, so that frame overwrote the key and un-retired the original — the assertion passed with the wake-up path fully broken. It now reuses the original token throughout, and reverting the guard above reddens it. * refactor(terminal): one i18n namespace per component, and two notes worth keeping The disconnected banner was renamed but kept reading five keys under the old component's namespace while its new strings sat under the new one — a component translating from two namespaces at once. Keys moved across all five locales; values and behaviour unchanged. Two comments earn their place. durablyBoundPtyIdForPane deliberately does NOT require the tab to still exist, unlike findTerminalTabIdForLeaf which must — adding the "missing" check there would make arbitration retire shells more eagerly, which is the opposite of what this change is for. And the respawn after the proof check in the direct-SSH catch is unreachable today by construction; saying so stops it reading as forgotten code. * fix(ssh): unknown PTY liveness is not proof of death hasPty is three-state and says so: null means the provider has not listed the host yet — ignorance, not death. The retry gate tested it with `!`, which reads null and false alike, so it ended recovery on ignorance. That state is not exotic: a reconnect builds a fresh provider with an empty set, which is exactly the moment rejected frames arrive. The attempt was deleted and no retry scheduled, and because the delivery token was already retired, no later frame could revive it — the pane stayed dark. It also short-circuited the park cooldown, since parkedAt was never set on an entry that no longer existed. Only an explicit false stops us now. The proven-exit clause still pins that. * fix(terminal): do not unbind a live shell the new-terminal spawn adopted While the card is up the pane keeps its durable binding on purpose, so a failed replacement can put the card back. But main resolves a stable pane's owner from that same binding: if the shell answers again between the failed reattach and the click, the spawn adopts it and returns its id. Nothing was replaced, and clearing the binding then unbound a shell that is live and attached to this very pane — leaving it running with no durable record of what it owns. Detected by identity: an id equal to the one we were replacing means adoption, not replacement. Suppressing adoption outright would need a new spawn option plumbed renderer -> transport -> IPC; the user-visible oddity that remains is getting the old shell back rather than a new one, which is benign next to orphaning it. * feat(ssh): record the host-attested shell identity on the lease A lease names a shell by ptyId, and ptyId alone cannot identify one: a replaced relay restarts its ids at pty-1, so the same id can name somebody else's shell. This adds the missing primitive — the incarnation the HOST attested — so a later change can tell "my shell" from "a different shell wearing its id". No behaviour changes yet; nothing reads the field. Only host-attested values are stored. The provider synthesizes a stand-in when a host reports none; that stand-in is first-write-wins and is dropped when provider state resets, so the same live shell can present a different one after a reconnect. Recording that would later read as a different shell and strand a live pane, so it is refused, and the synthesizer now shares the prefix constant with the predicate that rejects it. Leases are rebuilt field by field on load, so the normalizer had to learn the field too — adding it to the type alone drops it on every boot, which is how a fence ships silently permitting everything. The oracle reddens on exactly that. * fix(ssh): fence a recycled relay PTY id on the shell's own identity A reset relay restarts its ids at pty-1, so an id alone can name somebody else's shell: a pane still bound to pty-1 could attach to a shell another pane was already driving, and the two would share keystrokes and output. The guard that used to catch this compared the paneKey and tabId frozen at spawn. It was removed for good reason — a pane moved to another tab was refused its own live shell, and refused as "not found", which read as death and resumed the agent a second time. So it traded a cross-attach for a double resume. The incarnation is the shell's OWN identity, so it discriminates a recycled id without caring where the pane lives: both failures close at once. The relay refuses a mismatch and is deliberately not worded "not found" — that phrasing is what the client maps to an expired session, and expiry authorizes a respawn onto a shell this branch just proved is alive. Only a host-attested expectation is sent. The locally synthesized stand-in is not stable across reconnects and would refuse a pane its own shell. An older relay ignores the field and an older client sends none, so both stay permissive. The tab-keyed comparator is deleted rather than left dormant, and its suite is inverted onto the new identity: a recycled id is still refused, and a pane that moved tabs now attaches instead of being told its shell is gone. * fix(ssh): only an exit the relay watched may authorize a replacement A bare not-found became SSH_SESSION_EXPIRED, and expiry authorizes a respawn. But "the relay I asked cannot hand that id back" proves an exit only if that relay is the one that minted it — a replaced relay answers exactly this for shells still running under its predecessor. So after a relay restart, live orphaned shells were read as dead: ownership cleared, lease expired, and the agent resumed a second time onto a transcript its first process was still writing to. The relay now keeps what it actually observed. On a real exit, and on the liveness probe that finds a pid gone, it remembers {code, incarnation} in a bounded map and answers a later attach with SSH_PTY_EXITED instead of throwing that knowledge away. That is first-hand, same-process evidence, and it is the only answer that now maps to expiry. A remembered exit for a DIFFERENT incarnation is not an answer about the caller's shell, so a recycled id cannot report a stranger's death as its own. A crash loses the map, which correctly reads as no knowledge rather than as death. The carrier is the message text: the relay's error transport keeps only a message and a numeric code, so a structured payload would not survive. It is deliberately worded to avoid "not found", which older clients map to expiry. Cost, stated plainly: against a relay too old to remember exits, a genuinely dead shell is now unproven, so the pane offers the disconnected card instead of replacing itself. That is the affordance's purpose, and it is the safe direction. * fix(ssh): a remembered exit answers only the shell that asked for it Review found three ways the new proof could be believed too easily. A caller that names no shell was still handed a remembered exit. That is the same double resume in a new costume: relay A is killed leaving pty-1 alive and orphaned, relay B mints its own pty-1 and THAT one exits, and a pane carrying no recorded identity would be told its shell is gone — then replace a process still running. The expectation must now be present AND match. Panes with nothing to compare get the disconnected card, which is the direction that cannot lose work. The proof is also gated on the client declaring it understands it. What the host answers reaches clients that predate the reply, and one of those reads an unrecognised attach error as neither death nor recovery — a stranded pane. Older clients keep the wording they already act on; nothing is lost, because they could not have used the proof anyway. And the match is anchored on the whole grammar rather than the token, with the id and incarnation percent-encoded. A substring test would let any text that merely quotes the token stand in for the relay's own observation. Two callers that key on "already gone" now also accept a proven exit, so the liveness-probe reap does not burn a retry before reaching the same conclusion. The oracle for the first of these was itself vacuous: `toThrow` with a negated asymmetric matcher passes whenever anything throws at all. It now reads the thrown message, and reverting the guard reddens it. * fix(ssh): the liveness reap proves nothing to a caller who named another shell The remembered-exit route was fixed to require a present, matching expectation. The liveness probe is the other route to the same claim, and it still answered anyone: it fires when a pty EXISTS but its pid is gone, and under a given id a replacement relay may hold a shell that is not the caller's at all. Its death is then no evidence about a pane whose own shell may be running orphaned under the relay this one replaced — and the reply authorizes replacing it. Both routes now demand the same thing. The reap still happens either way, because a dead shell should be cleared whoever asked; only the answer depends on whose it was, and a caller who named a different shell is told exactly that. Found by asking whether the guard ordering was right, after the first fix closed only the half that had been reported. * fix(ssh): the client checks whose exit the proof is about Enforcement lived only on the host. But the host is the party whose answer is in question, and mixed versions are the normal state — a host that applies the rule loosely, or not at all, could hand back an exit for a shell the pane never owned and the client would replace a process that is still running. The incarnation travels in the proof precisely so the asking side can check it. A proof that cannot be tied to the shell this pane asked about is not proof, and falls through to the disconnected pane. A pane that knows no incarnation cannot verify anything, so it does not get to act on one either. Both halves now apply the same rule independently, which is what makes it hold across versions rather than only when both ends agree. * fix(ssh): fence the reconnect path too, not just the pane-driven restore There are two client attach routes and only one was fenced. The pane-driven restore goes through reattachSshPtySession, which was sending the shell identity; the relay session's own reattach — the one that reconnects EVERY known pty when a relay comes back — goes through attachForReconnect, which sent nothing. That is the wrong one to leave open. A relay coming back is exactly when ids have been reissued from pty-1, so the main reconnect was the likeliest place to attach somebody else's shell, and it was attaching by id alone. It now sends the identity the lease recorded, which is what the lease field added earlier was for; the two halves finally meet. Pane identity is still not sent — only the shell's own — so a pane that moved tabs is unaffected. It also declares exit-proof support, so a proven exit can reach the path that reattaches after a host restart. Callers with nothing extra to say keep the older call shape, so this does not churn every reconnect assertion in the suite over trailing undefineds. * fix(ssh): actually write the shell identity the reconnect fence reads The lease field was declared, preserved on load, and read on reconnect — and never written. Both spawn writers omitted it, so every lease carried only "pty-N", the reconnect always took the no-expectation path, and the fence added for it could not fire. The main reconnect went on attaching by id alone, which is the replaced-relay case the fence exists for. Worse than inert: a successful unfenced attach durably binds the pane to whatever answered, so the wrong identity would be recorded and carried forward. The host attests the identity at spawn and it was already in scope one line above both writers. The oracle for this had to pin the WRITE. Every other clause — the type, the loader, the reader, the reconnect forwarding — was green throughout, because each was correct in isolation; only nothing joined them. The test that covers the spawn now seeds a host incarnation and requires it on the persisted lease, and removing either writer reddens it. * refactor(ssh): one home for the exit-proof rule, and comments the house style allows AGENTS.md asks for brief non-obvious comments, one line where possible. Several of mine ran to five and eight lines of narrative on the relay's attach path, which is the one place a reviewer most needs to scan the branching quickly. They now say the same thing shorter. Both gone-paths in attach() had independently spelled out the rule that proof must name the caller's own shell. That duplication is what let the earlier fix close one and miss the other, so the condition is now a single predicate both ask — a change to the rule cannot reach one path and skip the other. The renderer kept its own copy of SSH_SESSION_EXPIRED while this branch created a shared home for exactly that token, whose whole reason for existing is that the two copies once disagreed about an identity mismatch and the renderer respawned a live shell. It imports the shared one now. Not changed, after checking: the per-generation recovery budget still does not reset on a successful reattach. Resetting it looks obviously right and is wrong — a flapping PTY alternates failure and success, and a covering test drives exactly that for forty rounds. The park cooldown already bounds the harm. * fix(ssh): keep the pane fence for clients that cannot name a shell Deleting the relay's pane-identity comparison disarmed the recycled-id guard for every client that has not upgraded. The relay is shared and host-side: one person updating a host would leave their colleagues attaching by id alone, with nothing checking it in either direction. It is back as a FALLBACK, used only when the caller sends no incarnation. A client that can name the shell is still fenced on that and still attaches after moving tabs; a client that cannot gets the older, coarser check it was already living with rather than none at all. The two clauses inverted when it was deleted are restored, because they send pane identity with no incarnation — the old-client shape, which should be refused. The moved-pane property they used to contradict is pinned separately by the clause that sends an incarnation, and reverting the override reddens it. * fix(ssh): a bare not-found must not retire a pane's owner Retiring a stable pane's owner authorizes a replacement that carries the pane's agent resume payload. Until this branch, an SSH reattach failure reached that decision as SSH_SESSION_EXPIRED, which the gone-check did not match, so it never fired for SSH. Retiring the not-found mapping changed that: the raw `PTY "" not found` now matches, so a replaced relay answering for a shell its predecessor is still running would retire the owner, fabricate an exit, and respawn with the resume payload — the double resume, reintroduced by the commit meant to prevent it. For an SSH pane the two proving answers are the relay's own observed exit and the expiry the reattach mints only after verifying that proof names this shell. A bare not-found is neither, and now propagates instead: the pane keeps its owner and surfaces as disconnected. Local and daemon ptys are unchanged — their provider owns its ptys, so absence really is proof. The shutdown paths that also use the gone-check are untouched: there, "not found" is the outcome being asked for. The covering test drove the dangerous shape directly — bare not-found, retire, respawn with `codex resume …`. It now drives the proof, and the bare not-found case is pinned beside it. * fix(ssh): retire a lease the relay proved dead Reattach failure left every record untouched, including the one answer that settles it. A shell the relay watched exit kept a live lease, so every later reconnect fanned out an attach for it — two attempts and a ten-second deadline each — and the set only grew, in a file written to disk. A proven exit now retires the record. `terminated`, not `expired`: expiry is the state the recovery grant reads, and retiring a record must not also authorise a replacement. Everything else is unchanged and still leaves the pane detached and recoverable, which is the point — a not-found is also what a replaced relay answers for shells its predecessor is still running. * fix(ipc): one import of the incarnation module, not two CI's code-quality plugins deny a second import of a module already imported in the same file; the pre-commit hook runs a different oxlint config and did not see it. Both failing checks were this: `verify` is a gate that only reported static analysis, with typecheck, tests and both package jobs already green. * fix(ssh): harden PTY reattach reliability * fix(persistence): fence duplicate lease rollback * test(pty): pin incarnation write fence * fix(ui): center narrow terminal recovery actions * fix(ssh): make the unreachable-pane state reachable, and its actions work The decisive cases for this affordance have sat at `fixme` because the state was not inducible: the card needs the SSH target CONNECTED while exactly one pane's attach fails without proving the shell gone, and every host-side fault takes the whole connection down instead. So the behaviour was only ever argued from reading, which is how several oracles here ended up green for the wrong reason. It is inducible now, using the fence this branch added for another purpose: the relay refuses an attach whose expected incarnation names a different shell, which is per-pty and leaves the connection healthy. Rewriting a pane's recorded identity while the app is closed reproduces it deterministically. Running it immediately found two defects that reading had not: The error toast painted over the card. It renders at z-50 in the same bottom strip and was suppressed only for the connection overlay, not for the pane's own card — so in the one state this affordance exists for, BOTH its actions were unclickable. "Start a new terminal" then did nothing at all: no shell, no host change, card straight back. It routes through the path that resolves the pane's owner and attaches it, and the owner is the very shell we cannot reach — so the attach fails and nothing is created. The action now refuses adoption, which is what makes it a creation. Nothing is killed; the old shell stays alive and unbound. Honest status: the gate is NOT green. The card now clears and the action runs, but shell creation is not yet observed reliably across runs. Committing so the oracle and both fixes are not lost; the remaining failure is the next work. * fix(ssh): a session id is the instruction to attach, so a refused adoption drops it Skipping stable-pane owner resolution in main was not enough: the action still sent the pane's recorded sessionId, and the provider reattaches on that before any owner logic runs. So "Start a new terminal" kept attaching the very shell it could not reach, and created nothing. The id is now dropped at the last gate before the IPC, where it cannot be reintroduced by a caller that forgets. Also records what running the gate has established so far, including the one defect still open: with both fixes in, the click still produces no spawn at all (visible=true launches=1 shells=1), while the main log over the same window shows only the pane's own restore retries. The evidence points at the action closure belonging to a superseded connection, not at the spawn path — so the next step is to instrument the handler rather than add a third spawn-path guard. * docs(ssh): locate the remaining defect — main re-derives the session id Instrumented the handler, the transport and main in one correlated run. The closure hypothesis was wrong: the handler runs, the connection is live, and the renderer half is correct — it sends no session id and asks for adoption to be refused. Main re-derives the id anyway, attaches the unreachable shell, and the spawn rejects, so the action resolves null and the card returns. That narrows it from "a renderer race" to one gate in main: createFreshShellForUnreachablePane covers only the early owner resolution, while a second site downstream still passes sessionId: owner.ptyId to the provider. Recorded with the evidence and the shortlist of call sites, plus the instruction to gate where the owner is CONSUMED rather than adding a third condition at a third producer — this is the same rule leaking at a third site, which is the signature this branch keeps producing. * fix(ssh): refuse adoption where the owner is consumed, not where it is derived The unreachable pane's "start a new terminal" still attached the shell it could not reach. The renderer was already correct — instrumenting handler, transport and main together showed it sending no session id and asking for adoption to be refused, while main re-derived the id anyway. The rule had been applied at the two places an owner is PRODUCED and missed at the one place it is CONSUMED: spawnForStablePane turns an owner into `sessionId` for the provider, which is what makes an attach an attach. Gating there closes it for every producer at once. The decisive E2E now passes: with the target connected and one pane unreachable, the action creates exactly one shell, leaves the unproven old shell running and unbound, and clears the card. Reverting the single condition reddens it. This rule leaked at three sites in a row and each fix was necessary while none was sufficient. The one that held was placed where the value is used. * refactor(terminal): the same-id guard now protects a new shell, not an adoption Refusing adoption removed the case this guard was written for. What remains is the opposite: a reset relay can reissue the old id to a genuinely NEW shell, and main has already bound the pane to it — so clearing by that id would unbind the shell just created. Same code, and it is still needed; the comment said the wrong reason, which is how the next reader deletes it. This also closes the planned "tell the user we recovered your terminal" work as obsolete: there is no silent recovery left to announce, because the action now always creates. --------- Co-authored-by: Orca --- .gitignore | 3 + config/reliability-gates.jsonc | 8 +- ...un-ssh-docker-maxsessions-identity-e2e.mjs | 41 + ...-docker-reconnect-pane-cardinality-e2e.mjs | 41 + ...run-ssh-docker-two-hosts-isolation-e2e.mjs | 41 + .../terminal-session-behavior-contract.md | 146 ++++ .../PR-13326-REPLACEMENT-DESIGN-2026-08-13.md | 192 ++++ .../PR-13326-SESSION-HANDOFF-2026-08-12.md | 823 ++++++++++++++++++ .../README.md | 265 ++++++ .../RESUME-PROMPT.md | 111 +++ .../binding-identity-design.md | 253 ++++++ .../counsel-design.html | 278 ++++++ .../design-brief-for-review.md | 186 ++++ .../design-explained.html | 235 +++++ .../design-final-detail.md | 256 ++++++ .../design-overview.md | 255 ++++++ .../finalized-design.html | 275 ++++++ .../github-overlap-query.graphql | 324 +++++++ .../goalposts.md | 683 +++++++++++++++ .../new-design-goalposts.md | 386 ++++++++ .../related-open-work.md | 341 ++++++++ .../report.html | 376 ++++++++ .../resume-plan.md | 275 ++++++ .../test-overview.md | 277 ++++++ package.json | 4 + src/main/daemon/daemon-pty-adapter.test.ts | 14 + src/main/daemon/daemon-pty-adapter.ts | 6 +- src/main/daemon/daemon-pty-router.test.ts | 27 + src/main/daemon/daemon-pty-router.ts | 22 +- .../degraded-daemon-pty-provider.test.ts | 35 + .../daemon/degraded-daemon-pty-provider.ts | 23 +- .../ssh-pty-output-intake-test-harness.ts | 2 +- .../pty-superseded-operation-fence.test.ts | 118 +++ src/main/ipc/pty.test.ts | 427 ++++++++- src/main/ipc/pty.ts | 305 +++++-- src/main/ipc/ssh-pty-output-intake.test.ts | 2 +- .../ssh-pty-output-model-migration.test.ts | 2 +- src/main/ipc/ssh.test.ts | 1 + .../persistence-ssh-lease-termination.test.ts | 152 ++++ src/main/persistence.test.ts | 3 + src/main/persistence.ts | 389 ++++++++- src/main/providers/pty-provider-contract.ts | 3 +- src/main/providers/ssh-pty-errors.ts | 26 +- ...pty-identity-mismatch-is-not-death.test.ts | 215 +++++ ...der-agent-session-create-operation.test.ts | 3 +- .../ssh-pty-provider-exit-race.test.ts | 27 + .../ssh-pty-provider-output-state.ts | 3 +- .../ssh-pty-provider-process-events.test.ts | 11 + ...-pty-provider-reattach-incarnation.test.ts | 30 +- src/main/providers/ssh-pty-provider.test.ts | 73 +- src/main/providers/ssh-pty-provider.ts | 35 +- .../providers/ssh-pty-session-reattach.ts | 94 +- src/main/runtime/orca-runtime.test.ts | 59 +- src/main/runtime/orca-runtime.ts | 63 +- src/main/runtime/orchestration/db.ts | 12 +- .../orchestration/lifecycle-reconciliation.ts | 15 +- .../orchestration-legacy-process-identity.ts | 11 +- ...h-pane-recovery-grant-reachability.test.ts | 157 ++++ .../terminal-list-stale-leaf-liveness.test.ts | 17 + ...kspace-namespace-terminal-identity.test.ts | 365 ++++++++ ...e-session-terminal-membership-authority.ts | 24 + src/main/ssh-pane-binding-partition.test.ts | 399 +++++++++ .../ssh-reattach-pane-cardinality.test.ts | 742 ++++++++++++++++ .../ssh-relay-orphan-abandon-paths.test.ts | 19 +- .../ssh/ssh-relay-reattach-exit-proof.test.ts | 358 ++++++++ ...ay-session-agent-hooks.integration.test.ts | 1 + .../ssh-relay-session-data-delivery.test.ts | 2 - .../ssh-relay-session-model-migration.test.ts | 2 - ...-relay-session-reattach-pane-fence.test.ts | 334 +++++++ ...elay-session-reconnect-incarnation.test.ts | 52 +- .../ssh-relay-session-recovery-races.test.ts | 4 +- ...sh-relay-session-rejected-delivery.test.ts | 173 +++- .../ssh-relay-session-terminal-error.test.ts | 1 + .../ssh/ssh-relay-session-test-fixtures.ts | 4 +- ...ost-delivery-isolation.integration.test.ts | 402 +++++++++ src/main/ssh/ssh-relay-session.test.ts | 77 +- src/main/ssh/ssh-relay-session.ts | 298 ++++--- src/preload/api-types.ts | 2 + src/preload/index.ts | 1 + src/relay/pty-handler.test.ts | 275 ++++-- src/relay/pty-handler.ts | 112 ++- .../components/terminal-pane/TerminalPane.tsx | 24 +- .../TerminalPaneDisconnectedBanner.test.tsx | 164 ++++ .../TerminalPaneDisconnectedBanner.tsx | 206 +++++ ...minalRemoteRuntimeReconnectBanner.test.tsx | 33 - .../TerminalRemoteRuntimeReconnectBanner.tsx | 72 -- .../terminal-pane/pty-connection.test.ts | 459 +++++++++- .../terminal-pane/pty-connection.ts | 185 +++- .../terminal-pane/pty-transport-types.ts | 9 + .../terminal-pane/pty-transport.test.ts | 23 + .../components/terminal-pane/pty-transport.ts | 69 +- .../reattach-failure-classification.test.ts | 107 +++ .../reattach-failure-classification.ts | 26 + src/renderer/src/i18n/locales/en.json | 14 +- src/renderer/src/i18n/locales/es.json | 2 +- src/renderer/src/i18n/locales/ja.json | 2 +- src/renderer/src/i18n/locales/ko.json | 2 +- src/renderer/src/i18n/locales/zh.json | 2 +- src/shared/pty-incarnation.ts | 13 + src/shared/ssh-pty-failure-tokens.ts | 76 ++ src/shared/ssh-types.ts | 8 + src/shared/stable-pane-id.ts | 12 + ...ersion-reattach-authorization.unit.test.ts | 156 ++++ .../cross-version-terminal-wire.unit.test.ts | 101 +++ .../reattach-failure-publication-skew.ts | 290 ++++++ .../reattach-recovery-authorization-skew.ts | 333 +++++++ .../versioned-terminal-wire.ts | 18 +- .../daemon-restart-session-liveness.spec.ts | 424 +++++++++ .../helpers/daemon-shell-process-identity.ts | 151 ++++ .../docker-ssh-host-probe-observations.ts | 102 +++ .../docker-ssh-relay-host-partition.ts | 35 + .../e2e/helpers/docker-ssh-relay-processes.ts | 31 + .../helpers/docker-ssh-relay-remote-ptys.ts | 149 ++++ .../docker-ssh-relay-sshd-session-cap.ts | 64 ++ tests/e2e/helpers/docker-ssh-relay-stall.ts | 70 ++ tests/e2e/helpers/docker-ssh-relay-target.ts | 21 +- .../docker-ssh-relay-workspace-census.ts | 111 +++ .../docker-ssh-remote-shell-identity.ts | 112 +++ .../helpers/docker-ssh-remote-shell-pid.ts | 64 ++ tests/e2e/helpers/paired-electron-client.ts | 6 +- .../helpers/remote-pane-durable-session.ts | 58 ++ .../helpers/remote-pane-launch-transcript.ts | 68 ++ .../e2e/helpers/ssh-pane-incarnation-fault.ts | 74 ++ .../e2e/helpers/ssh-remote-pty-lease-file.ts | 62 ++ tests/e2e/helpers/unbound-remote-pty-lease.ts | 109 +++ ...-terminal-restart-binding-identity.spec.ts | 325 +++++++ tests/e2e/paired-runtime-version-skew.spec.ts | 217 +++++ .../ssh-disconnected-pane-affordance.spec.ts | 391 +++++++++ ...azy-discovery-skipped-host-restart.spec.ts | 661 ++++++++++++++ ...ssions-remote-pid-binding-identity.spec.ts | 739 ++++++++++++++++ ...attach-does-not-resume-agent-twice.spec.ts | 244 ++++++ ...pane-cardinality-across-partitions.spec.ts | 349 ++++++++ .../ssh-reconnect-pane-cardinality.spec.ts | 424 +++++++++ tests/e2e/ssh-two-hosts-isolation.spec.ts | 360 ++++++++ .../e2e/ssh-unreachable-pane-recovery.spec.ts | 191 ++++ ...playwright.daemon-liveness.local.config.ts | 13 + 136 files changed, 19349 insertions(+), 786 deletions(-) create mode 100644 config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs create mode 100644 config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs create mode 100644 config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs create mode 100644 docs/reference/terminal-session-behavior-contract.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-REPLACEMENT-DESIGN-2026-08-13.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-SESSION-HANDOFF-2026-08-12.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/README.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/RESUME-PROMPT.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/binding-identity-design.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/counsel-design.html create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md create mode 100644 docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md rename src/main/ipc/{ => __tests__}/ssh-pty-output-intake-test-harness.ts (97%) create mode 100644 src/main/ipc/pty-superseded-operation-fence.test.ts create mode 100644 src/main/persistence-ssh-lease-termination.test.ts create mode 100644 src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts create mode 100644 src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts create mode 100644 src/main/runtime/workspace-namespace-terminal-identity.test.ts create mode 100644 src/main/ssh-pane-binding-partition.test.ts create mode 100644 src/main/ssh-reattach-pane-cardinality.test.ts create mode 100644 src/main/ssh/ssh-relay-reattach-exit-proof.test.ts create mode 100644 src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts create mode 100644 src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts create mode 100644 src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx create mode 100644 src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.tsx delete mode 100644 src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx delete mode 100644 src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.tsx create mode 100644 src/renderer/src/components/terminal-pane/reattach-failure-classification.test.ts create mode 100644 src/renderer/src/components/terminal-pane/reattach-failure-classification.ts create mode 100644 src/shared/ssh-pty-failure-tokens.ts create mode 100644 tests/e2e/cross-version-wire/cross-version-reattach-authorization.unit.test.ts create mode 100644 tests/e2e/cross-version-wire/reattach-failure-publication-skew.ts create mode 100644 tests/e2e/cross-version-wire/reattach-recovery-authorization-skew.ts create mode 100644 tests/e2e/daemon-restart-session-liveness.spec.ts create mode 100644 tests/e2e/helpers/daemon-shell-process-identity.ts create mode 100644 tests/e2e/helpers/docker-ssh-host-probe-observations.ts create mode 100644 tests/e2e/helpers/docker-ssh-relay-host-partition.ts create mode 100644 tests/e2e/helpers/docker-ssh-relay-remote-ptys.ts create mode 100644 tests/e2e/helpers/docker-ssh-relay-sshd-session-cap.ts create mode 100644 tests/e2e/helpers/docker-ssh-relay-stall.ts create mode 100644 tests/e2e/helpers/docker-ssh-relay-workspace-census.ts create mode 100644 tests/e2e/helpers/docker-ssh-remote-shell-identity.ts create mode 100644 tests/e2e/helpers/docker-ssh-remote-shell-pid.ts create mode 100644 tests/e2e/helpers/remote-pane-durable-session.ts create mode 100644 tests/e2e/helpers/remote-pane-launch-transcript.ts create mode 100644 tests/e2e/helpers/ssh-pane-incarnation-fault.ts create mode 100644 tests/e2e/helpers/ssh-remote-pty-lease-file.ts create mode 100644 tests/e2e/helpers/unbound-remote-pty-lease.ts create mode 100644 tests/e2e/local-terminal-restart-binding-identity.spec.ts create mode 100644 tests/e2e/paired-runtime-version-skew.spec.ts create mode 100644 tests/e2e/ssh-disconnected-pane-affordance.spec.ts create mode 100644 tests/e2e/ssh-lazy-discovery-skipped-host-restart.spec.ts create mode 100644 tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts create mode 100644 tests/e2e/ssh-reattach-does-not-resume-agent-twice.spec.ts create mode 100644 tests/e2e/ssh-reconnect-pane-cardinality-across-partitions.spec.ts create mode 100644 tests/e2e/ssh-reconnect-pane-cardinality.spec.ts create mode 100644 tests/e2e/ssh-two-hosts-isolation.spec.ts create mode 100644 tests/e2e/ssh-unreachable-pane-recovery.spec.ts create mode 100644 tests/playwright.daemon-liveness.local.config.ts diff --git a/.gitignore b/.gitignore index 8fb2d756f1c..f5ffbfd5f3c 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,9 @@ docs/** !docs/reference/linux-glibc-compatibility.md !docs/reference/relay-grace-time-reconfiguration.md !docs/reference/remote-wire-compatibility.md +!docs/reference/terminal-session-behavior-contract.md +!docs/reference/terminal-session-correctness-handoff-2026-08-07/ +!docs/reference/terminal-session-correctness-handoff-2026-08-07/** !docs/reference/renderer-agent-status-performance.md !docs/reference/windows-setup-shell.md !docs/reference/worktree-scan-fingerprint.md diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index eab6a047bc6..81f587046f0 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -6801,7 +6801,7 @@ "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/startup/ssh-startup-reconnect.test.ts src/renderer/src/lib/resolved-worktree-execution-host.test.ts src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts src/renderer/src/runtime/web-session-terminal-handle-events.test.ts src/renderer/src/store/slices/terminal-pty-identity-replacement.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport.test.ts", - "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/store/slices/direct-ssh-terminal-retry.test.ts src/renderer/src/store/slices/direct-ssh-pane-detach-ledger.test.ts src/renderer/src/store/slices/direct-ssh-terminal-recovery.test.ts src/renderer/src/store/slices/direct-ssh-terminal-workspace-scope.test.ts src/renderer/src/store/slices/terminals-hydration.test.ts src/renderer/src/store/slices/repos-ssh-host-reconciliation.test.ts src/renderer/src/hooks/direct-ssh-reconnect-coordinator.test.ts src/renderer/src/hooks/direct-ssh-host-hydration.test.ts src/renderer/src/hooks/direct-ssh-state-routing.test.ts src/renderer/src/hooks/remote-workspace-target-sync.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/repos-remote.test.ts src/main/ipc/ssh.test.ts src/main/ipc/worktrees.test.ts src/main/runtime/public-ssh-state.test.ts src/main/ssh/ssh-connection-manager.test.ts src/main/ssh/ssh-connection.test.ts src/main/ssh/ssh-provider-authority.test.ts src/preload/ssh-authority-forwarding.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/runtime/runtime-environment-ssh-state.test.ts src/shared/ssh-retained-payload-admission.test.ts src/shared/ssh-types.test.ts --reporter=dot", "pnpm exec electron-vite build --mode e2e", @@ -6822,7 +6822,7 @@ "src/renderer/src/components/terminal-pane/pty-transport.test.ts", "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts", "src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts", - "src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx", + "src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx", "src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts", "src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts", "src/renderer/src/runtime/web-session-terminal-handle-events.test.ts", @@ -6904,7 +6904,7 @@ ] }, { - "file": "src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx", + "file": "src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx", "assertions": [ "automatic recovery renders as a quiet bounded status without a button", "the disconnected state exposes one explicit Reconnect action" @@ -7179,7 +7179,7 @@ "date": "2026-07-21", "runner": "local", "platform": "macos", - "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts", "result": "passed", "durationSeconds": 5, "summary": "Eleven fault-injection and recovery-UI files and 127 tests passed, covering suspended heartbeat clocks, stale socket, PTY, and create generations, canonical pre-ready close recovery with one replacement subscription, cold and repeated PTY reattachment, authoritative health, bounded PTY and terminal-create recovery, post-probe timeout clipping, manually retryable create cutoff, accurate capability-probe failures, same-PTY manual reconnect, pane-state cleanup, fatal error deduplication, stable create identity, cross-process PTY adoption, and fail-closed legacy inventory." diff --git a/config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs b/config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs new file mode 100644 index 00000000000..2f232420531 --- /dev/null +++ b/config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs @@ -0,0 +1,41 @@ +import { spawnSync } from 'node:child_process' + +const rawExtraArgs = process.argv.slice(2) +const extraArgs = rawExtraArgs[0] === '--' ? rawExtraArgs.slice(1) : rawExtraArgs +const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' +const env = { + ...process.env, + ORCA_E2E_SSH_DOCKER: '1' +} + +// Why: Node's CVE-2024-27980 hardening rejects .cmd spawns without shell on Windows. +const spawnOptions = { + stdio: 'inherit', + env, + shell: process.platform === 'win32' +} + +const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], spawnOptions) + +if (runtime.status !== 0) { + process.exit(runtime.status ?? 1) +} + +const result = spawnSync( + pnpm, + [ + 'exec', + 'playwright', + 'test', + 'tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts', + '--config', + 'tests/playwright.config.ts', + '--project', + 'electron-headless', + '--workers=1', + ...extraArgs + ], + spawnOptions +) + +process.exit(result.status ?? 1) diff --git a/config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs b/config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs new file mode 100644 index 00000000000..cde82379c1f --- /dev/null +++ b/config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs @@ -0,0 +1,41 @@ +import { spawnSync } from 'node:child_process' + +const rawExtraArgs = process.argv.slice(2) +const extraArgs = rawExtraArgs[0] === '--' ? rawExtraArgs.slice(1) : rawExtraArgs +const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' +const env = { + ...process.env, + ORCA_E2E_SSH_DOCKER: '1' +} + +// Why: Node's CVE-2024-27980 hardening rejects .cmd spawns without shell on Windows. +const spawnOptions = { + stdio: 'inherit', + env, + shell: process.platform === 'win32' +} + +const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], spawnOptions) + +if (runtime.status !== 0) { + process.exit(runtime.status ?? 1) +} + +const result = spawnSync( + pnpm, + [ + 'exec', + 'playwright', + 'test', + 'tests/e2e/ssh-reconnect-pane-cardinality.spec.ts', + '--config', + 'tests/playwright.config.ts', + '--project', + 'electron-headless', + '--workers=1', + ...extraArgs + ], + spawnOptions +) + +process.exit(result.status ?? 1) diff --git a/config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs b/config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs new file mode 100644 index 00000000000..45c3911a3b8 --- /dev/null +++ b/config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs @@ -0,0 +1,41 @@ +import { spawnSync } from 'node:child_process' + +const rawExtraArgs = process.argv.slice(2) +const extraArgs = rawExtraArgs[0] === '--' ? rawExtraArgs.slice(1) : rawExtraArgs +const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' +const env = { + ...process.env, + ORCA_E2E_SSH_DOCKER: '1' +} + +// Why: Node's CVE-2024-27980 hardening rejects .cmd spawns without shell on Windows. +const spawnOptions = { + stdio: 'inherit', + env, + shell: process.platform === 'win32' +} + +const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], spawnOptions) + +if (runtime.status !== 0) { + process.exit(runtime.status ?? 1) +} + +const result = spawnSync( + pnpm, + [ + 'exec', + 'playwright', + 'test', + 'tests/e2e/ssh-two-hosts-isolation.spec.ts', + '--config', + 'tests/playwright.config.ts', + '--project', + 'electron-headless', + '--workers=1', + ...extraArgs + ], + spawnOptions +) + +process.exit(result.status ?? 1) diff --git a/docs/reference/terminal-session-behavior-contract.md b/docs/reference/terminal-session-behavior-contract.md new file mode 100644 index 00000000000..e345f36c8c2 --- /dev/null +++ b/docs/reference/terminal-session-behavior-contract.md @@ -0,0 +1,146 @@ +# Terminal session behavior contract + +Properties every terminal-session change must preserve. They describe +observable behavior, not mechanism, so a test written against them stays valid +under any implementation. + +Oracles for the STA-3077 properties live in +`src/main/ssh-reattach-pane-cardinality.test.ts`. + +## Why this exists + +Reconnecting an SSH-backed workspace used to add panes the user never opened, +and the remote host accumulated shells nobody was using — one report went from +2 to 19 to 20 relay PTYs across three reconnects. The failures in this class +share one shape: **something absent was treated as something dead**, or a +reattach was allowed to create what it should only have bound. + +## A. Unknown is not dead + +A disconnect, timeout, absent inventory entry, or failed inspection means +_unresolved_. None of them may terminate a PTY, release a binding, or authorize +cleanup. + +**A5** — a timer may never be the _sole cause_ of a destructive action. A timer +that bounds a wait or triggers re-verification is fine; a timer that kills is +not. Retry count never decides identity or liveness. + +This is deliberately weaker than "no timers". Recovery budgets and scratch-file +age-gates are correct code and must stay. + +Two patterns make a retention bound safe, and both beat shortening it: + +- measure **process time**, not wall clock, so a suspended laptop does not + burn the budget; +- gate aggressive reclamation on an **independent observation** — a second + consecutive scan, or another client completing a handshake — so the clock + bounds a wait while evidence authorizes the act. + +**B** — respawn requires proof. A failure that does not prove the session is +gone is unresolved: leave the shell running and keep the binding. An +unrecognized failure must never become a respawn, because a duplicated shell +is worse than a stalled one. + +## B. Attach binds; it never creates + +An ambiguous, unavailable, or rejected attach does not spawn a shell and does +not grow topology — no minted tab, no minted leaf, no split root, no minted +layout. Only an explicit create for a fresh pane may spawn. + +Corollary: after a recovery, keystrokes cannot land on a different shell than +the one that was visible before it. + +## C. Exact-binding fencing + +Mutating operations name the binding captured before the first await and are +rejected when any component no longer matches. A stale operation cannot reach a +later pane generation, a later PTY incarnation reusing the same id, or a +restarted owner. Output and exit arriving under a stale binding are dropped. + +The store already enforces this via `persistPtyBinding`'s `expectedBinding` +compare-and-swap and the per-repo topology revision fence. + +## D. Cardinality + +One pane owns at most one live PTY binding, and one PTY incarnation belongs to +at most one pane. Lease identity must therefore include the pane, not just the +transport id — keying on `(targetId, ptyId)` alone is what let predecessors +accumulate. + +Superseding a lease marks it expired; it does not kill the remote process, +because losing a lease is not proof the shell died. + +## E. Ordering and loss + +Output is delivered in PTY order. A gap is resolved by resnapshot, never by +silently continuing. A correctness-bearing effect (exit, command completion, +review link, bell) is neither lost while the renderer is unmounted nor applied +twice after a replay. + +Bounded replay plus idempotent application by event key satisfies this. A +durable per-consumer delivery cursor is not required, and is not currently +used. + +**Lifecycle is state, not an event.** An exit that happens while nothing is +attached is not recovered from a delivery channel — it is read during the +attach handshake. The attach reply should carry whether the session is alive +or exited, its exit code, and the resume offset. That is what removes the need +to guarantee delivery of an exit _event_, and it is why no journal is required +here. + +Where a resume offset cannot be served, advance the offset explicitly by the +size of the gap and persist the running total, so "output was lost here" stays +a queryable fact rather than a silent discontinuity. + +## F. Isolation + +A failure on one host cannot affect another host; a failure in one workspace +cannot affect another on the same connection. There is no process-wide failure +fence. + +## G. Liveness + +Every fail-closed path needs a bounded, user-reachable recovery. A workspace +must never reach a state that permanently refuses terminal operations with no +operator escape — in particular, backpressure from a consumer that is gone +forever must be releasable without that consumer's cooperation. + +Unavailable transports retry with bounded backoff and bounded memory. + +## H. Compatibility + +Clients and hosts update independently. A capability is negotiated or the newer +side degrades; a required operation is never silently dropped. New stream +opcodes must be capability-negotiated, because decoders drop unknown opcodes +silently. Nothing is gated on an app build id. + +See [`remote-wire-compatibility.md`](./remote-wire-compatibility.md). + +## I. Cost + +No new work on the input hot path and none per output frame. Pane create and +close latency must not regress, including over a high-latency SSH link. No +startup-blocking probe of a host nothing has asked for. + +--- + +## Test the call site, not the capability + +A guard that exists and is never passed is indistinguishable from no guard. +`mayCreate` was added to `persistPtyBinding`, was correct, and had no +production caller for several commits — every store-level test passed the whole +time, because they called the store directly. + +So for anything that refuses a destructive action, pin the **caller**: assert +that the reattach path passes the refusal, not merely that the store honours it +when asked. The same applies to a classifier — assert the branch that consumes +its verdict is reachable, since a guard behind an unreachable `catch` is dead. + +## Deliberately not required + +A durable event journal, per-consumer cumulative delivery cursors, and +cryptographic device principals are **not** required by any property above. +Each would need to earn its place by being the only way to satisfy one of them. + +Principals, in particular, exist only to key cursors: without cursors there is +nothing for them to key. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-REPLACEMENT-DESIGN-2026-08-13.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-REPLACEMENT-DESIGN-2026-08-13.md new file mode 100644 index 00000000000..0e5eae53bc4 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-REPLACEMENT-DESIGN-2026-08-13.md @@ -0,0 +1,192 @@ +# Design: make "Start a new terminal" a replacement, not a spawn + +Supersedes the mechanism in `PR-13326-SESSION-HANDOFF-2026-08-12.md` §"Approved bounded redesign". +Keeps its **properties**; replaces several of its **mechanisms** with smaller ones, and adds the +thing it was missing — a way to actually run the decisive test. + +## 1. The defect, stated once + +"Start a new terminal" is routed through a path whose job is *"resolve this pane's owner and attach +it; spawn only if there is none."* Creation is that path's fallback, not its purpose. The pane's +durable binding still names the old shell at click time, so the path resolves it and adopts it. + +I confirmed this first-hand earlier in the branch: commit `ab6ce176f3f` detects adoption after the +fact (`freshPtyId === sessionId`) and stops the callback from unbinding the live shell. Its own +message records that suppressing the adoption itself was left undone. So the adoption is real, and +the current code only contains its worst consequence. + +## 2. Why every previous fix leaked + +Each attempt expressed "do not adopt" as **another condition inside the adopting path**. Conditions +are forgettable and order-dependent, and this branch has three shipped proofs of that failure mode: + +- a guard centralised into a publisher that returned `void`, so four callers paired a conditional + publish with an unconditional `return`; +- the same rule applied to the tombstone branch and missed on the liveness branch beside it; +- the incarnation fence sent on the pane-driven route and not on the reconnect route. + +Every one was "the rule exists, but this path does not ask it." A fourth condition in the same path +is the same bet again. + +## 3. The design, and why it removes the defect + +### P1 — Replacement is a separate operation that cannot adopt + +Add a main-owned entry that performs replacement only. It contains **no owner-resolution call**: it +does not read a stable-pane owner, does not accept `sessionId`/`attachOnly`, and constructs provider +options from a narrow main-owned allowlist (dimensions + validated pane/workspace identity). + +*Why this fixes it:* adoption stops being a branch that must be avoided and becomes code that is not +present. A function with no owner-resolution cannot adopt for the same reason a function with no +socket cannot make a network call. This converts "remember to skip adoption" (liveness) into "there +is nothing here to adopt with" (structure). + +*Falsifiable by:* route the action back through generic spawn — the oracle in P0 must redden. + +### P2 — Ownership moves in one durable commit, keyed on exact incarnation + +One Store method mutates and flushes once: exact-CAS the old binding and the old active lease, +install the new binding + incarnation + attached lease, expire the exact old lease **without +touching its process**, bump the topology/lease versions, flush. + +*Why:* ownership lives in the durable binding and the lease. Moved separately, there is a window +where a racing reconnect sees a half-moved pane and restores the predecessor. One commit means the +window does not exist rather than being narrow. + +*Why exact incarnation:* the relay reissues `pty-N` from 1 after a reset, so any CAS keyed on ptyId +alone can match a different process. This branch already proved that and already carries the +primitive: host-attested `SshRemotePtyLease.incarnationId`, the relay-side `expectedIncarnationId` +fence, and `isRelayAttestedPtyIncarnationId` filtering synthesised values. + +*Falsifiable by:* split the commit in two flushes and drive a reconnect between them. + +### P3 — Ambiguity leaks, never kills + +Provider/relay shutdown identifies a PTY by id only. After a dispatched spawn whose outcome is +ambiguous, we cannot prove which incarnation a shutdown would reach. So: clean up a failed new shell +only when its exact incarnation is provable; otherwise leave it running and unbound for the existing +cleanup surface to reap. + +*Why:* the costs are asymmetric. Leaking costs an orphan shell that a user can already list and +terminate. Killing the wrong incarnation destroys a live agent session — the exact harm this whole +program exists to prevent. + +*Falsifiable by:* make the ambiguous path call shutdown; the oracle asserting zero kill RPCs on an +unprovable outcome must redden. + +### P4 — The successor is invisible until it is the owner + +Spawn the replacement hidden and install its output handlers only after P2 commits. + +*Why:* a new shell reusing the old raw id could otherwise deliver bytes into the predecessor's +model. Ordering removes the window without a new quarantine subsystem. + +### P5 — Stale renderer work does nothing + +Guard every completion on: not disposed, current transport identity, current action generation. A +stale callback must not clear a binding, republish the card, or kill a result. Main owns the durable +transition, so the renderer never clears the old binding on success. + +## 4. What I am deliberately NOT building, and the risk of each + +The handoff prescribes more machinery. I judge these unnecessary for this defect; each is a +reversible decision if an oracle shows otherwise. + +| Prescribed | Decision | Reasoning | Risk if I am wrong | +|---|---|---|---| +| Opaque ticket bound to ~11 fields (sender, both generations, worktree, tab, leaf, pane key, both ids, both incarnations, expiry, 5-state machine) | **Replace** with an operation id + expected-owner `{ptyId, incarnationId}` CAS | The renderer is not a trust boundary — it can already open terminals. The ticket's real jobs are staleness and idempotency, which the CAS and op id give directly. Eleven bindings add eleven invalidation paths, and invalidation bugs are the same class we are trying to leave | A forged/stale request creates one blank shell for a pane the user owns — recoverable, and the CAS still refuses to move ownership off an unexpected incumbent | +| A new shared owner mutex | **Reuse** `paneSpawnReservationsByOwnerKey` (already per-pane) and the runtime pane-create claim | Serialisation already exists at exactly this granularity; a second lock invites lock-order bugs | Two operations interleave on one pane; the P2 CAS still refuses the loser | +| A quarantine subsystem for provider output | **Reuse** hidden spawn + install-after-commit ordering | Achieves the same property with existing options | Bytes reach the wrong model; caught by the P4 oracle | + +## 5. The part the previous session was missing: the decisive test can now run + +`tests/e2e/ssh-disconnected-pane-affordance.spec.ts` holds the decisive case at `test.fixme` because +**the state was not inducible**: it needs the SSH target CONNECTED while exactly one pane's +`pty.attach` fails with an error that does not prove the shell gone. Host faults tried (SIGSTOP the +relay) take the whole connection down and produce the connection overlay instead. + +That state is now inducible, by a mechanism this branch added for another reason. The relay refuses +an attach whose `expectedIncarnationId` names a different shell, with +`PTY "" identity mismatch` — per-pty, connection healthy, and explicitly not proof of death +(`SSH_PTY_IDENTITY_MISMATCH` never grounds a respawn). + +So the test can seed the pane's lease with an incarnation that is not the host's, reconnect, and get +exactly the state the banner needs — deterministically, with no timing fault. + +**This is the highest-value item in the plan and it comes first.** The previous session looped +because every claim was adjudicated by reading code; with the oracle running, the design is +*demonstrated* instead of argued, and a regression reddens instead of being found in review. + +## 6. Goalposts + +Each is a behaviour with a stated mutation that must redden it. G1 is a prerequisite for believing +any of the others. + +| # | Goalpost | Mutation that must redden it | +|---|---|---| +| **G1** | The decisive E2E runs unskipped: with the target connected, one pane unreachable, clicking "Start a new terminal" yields exactly one new remote shell, the pane count is unchanged, and the host transcript shows no second agent launch | Remove the induction; the test can no longer reach the banner | +| **G2** | The action never adopts: the pane's shell id/incarnation after the click differs from before, and the new shell has no command, resume metadata, launch token, agent, or startup delivery | Route the action back through generic `pty:spawn` | +| **G3** | Ownership moves atomically: a reconnect racing the replacement cannot restore the predecessor; binding and lease agree at every observable point | Split the durable commit into two flushes | +| **G4** | Ambiguity never kills: on an unprovable spawn outcome, zero shutdown RPCs are issued and neither process dies | Call ID-only shutdown on the ambiguous path | +| **G5** | The successor is silent until it owns the pane: no provider bytes reach the predecessor's model, including when the raw id is reused | Install output handlers before the commit | +| **G6** | Stale renderer work is inert: a disposed or superseded callback clears no binding, republishes no card, kills no result | Drop the generation guard on the completion path | + +## 7. Sequence + +0. Remove the rejected six-file WIP (targeted edits, not a broad restore — the worktree is dirty). +1. **G1** — induction + un-`fixme`. Nothing else is believable until this runs. +2. **G2** — the separate replacement operation. +3. **G3** — the single durable commit. +4. **G4/G5/G6** — ambiguity, ordering, staleness. +5. Adversarial review on both models, then push. + +## 8. Stop conditions + +- If G1 cannot be made to run, stop and report. Building 2–6 without it repeats the previous loop. +- If exact cleanup cannot be achieved within existing local/provider contracts, take the leak (P3). + Do **not** add `expectedIncarnationId` to remote shutdown — that is a separate, capability- + negotiated wire change with its own review. + +--- + +## 9. Findings from running the gate (2026-08-13) + +The oracle in §5 works: the state is inducible and the card appears reliably. Running it found +three things that reading had not. + +**Fixed — the card's buttons were unclickable.** `TerminalErrorToast` renders at `z-50` in the same +bottom strip and was suppressed only for the connection overlay, not for the pane's own card. In +the one state this affordance exists for, both actions were covered. `TerminalPane.tsx` now also +suppresses it while the active pane shows the card. + +**Fixed — a session id is the instruction to attach.** "Start a new terminal" was still sending the +pane's recorded `sessionId`, which routes straight to reattach before any pane-owner logic runs, so +the action attached the very shell it could not reach and created nothing. Skipping owner +resolution in main was NOT enough on its own; the id had to be dropped at the last gate before the +IPC (`admittedSessionId` in `pty-transport.ts`). + +**OPEN — the action produces no spawn at all.** With both fixes in, the measured result after +clicking is still `visible=true launches=1 shells=1`: the card clears briefly, no shell is created, +and the card returns. The main log over the same window shows only the pane's own restore retries +(three `spawn() called with sessionId=…` → `identity mismatch`), never a fresh spawn. + +The evidence points at the action closure rather than the spawn path: the pane retries its restore +while the card is up, each retry republishes the card, and the button the test clicks appears to +belong to a superseded connection whose `startFreshSpawn` no longer does anything. That is the +"stale renderer work" hazard from the original handoff, showing up as a dead button rather than as +a wrong write. + +**RESOLVED.** Instrumenting handler, transport and main in one correlated run disproved the +closure hypothesis: the handler ran, the connection was live, and the renderer half was already +correct — it sent no session id and asked for adoption to be refused. Main re-derived the id anyway. + +The rule had been applied at the two places an owner is PRODUCED and missed at the one place it is +CONSUMED: `spawnForStablePane` turns an owner into `sessionId` for the provider, which is what makes +an attach an attach. Gating there — `if (args.owner && !args.refuseAdoption)` — closes it for every +producer at once, present and future. + +The gate now passes, and reverting that single condition reddens it. + +Lesson worth keeping: this rule leaked at three sites in a row (early owner resolution, the +transport's session id, the owner consumption). Each fix was necessary and none was sufficient. The +one that finally held was the one placed where the value is USED rather than where it is derived. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-SESSION-HANDOFF-2026-08-12.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-SESSION-HANDOFF-2026-08-12.md new file mode 100644 index 00000000000..bd4900ddf52 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/PR-13326-SESSION-HANDOFF-2026-08-12.md @@ -0,0 +1,823 @@ +# STA-3077 / PR #13326 session handoff + +> **Status:** paused at an approved bounded redesign; not release-ready. +> +> **Review/fix round:** 6. +> +> **Do not continue the current six-file implementation.** It is a rejected, +> uncommitted grant-map approach. Remove it with `apply_patch`, then implement +> the dedicated main-owned replacement transaction described below. + +This is the continuation entrypoint for the current PR. It records the whole +session at the point the user asked to pause, including what shipped to the PR +branch, what failed review, the agreed design boundary, and the next safe work. + +## Executive summary + +The session started with STA-3077's SSH reconnect failure class: reconnects +could graft panes the user never created, accumulate remote shells, fabricate +process death from an attach failure, or let stale identities affect a +successor. The branch now contains extensive fixes, tests, design records, and +two user-visible recovery actions for an unreachable pane. + +The pushed candidate is still incorrect in one important action. “Start a new +terminal” invokes generic `pty:spawn` while the old durable pane binding remains. +Main can adopt that old owner, so the action can reopen the old shell instead of +creating the one blank shell the user requested. + +An attempted six-file fix added `freshSpawnForUnreachablePtyId` to generic spawn +and a process-global grant map. Three independent read-only audits rejected it. +The common verdict was that replacement must be a dedicated local transaction +owned by Electron main, with exact incarnation identity, one durable Store +commit, runtime ownership transfer, output quarantine, reconnect fencing, and +renderer generation guards. + +The user approved this bounded redesign. It applies only to the broken +unreachable-pane replacement operation. It is not authority to restart the +earlier whole-system architecture effort. + +## Repository checkpoint + +Verified before this handoff was written: + +| Item | Value | +| ----------------------------------- | ---------------------------------------------------- | +| Worktree | `/Users/nwparker/orca/workspaces/orca/eye-React-185` | +| Local branch | `nwparker/sta-3077-reattach-pane-cardinality` | +| HEAD | `df6edfcebd2ea401a6feda1fccf31189e57eceda` | +| HEAD subject | `fix(ui): center narrow terminal recovery actions` | +| Actual PR | `stablyai/orca#13326` | +| Actual PR ref | `origin/nwparker/sta-3077-d-affordance-and-review` | +| Actual PR ref SHA | exactly `df6edfcebd2ea401a6feda1fccf31189e57eceda` | +| Configured upstream | `origin/nwparker/sta-3077-reattach-pane-cardinality` | +| Configured upstream SHA | `518dac34415e0f67883461c3f58e5586209c7f24` | +| Local `origin/main` | `889dd4c7d2fe255aac552e4dcc91204279dc022c` | +| Merge base with local `origin/main` | `58a926170cd99baf8920a83a86799bfbd307d735` | +| Local `origin/main...HEAD` | 10 commits on main / 109 commits on HEAD | + +The configured upstream is stale. Its `[ahead 387]` status is not the PR's +push state. The PR ref is exactly at HEAD. When the work is genuinely ready, +push explicitly: + +```bash +git push origin HEAD:nwparker/sta-3077-d-affordance-and-review +``` + +Before this document, the worktree had six modified tracked files, no untracked +files, and a `+163/-11` uncommitted diff. `git diff --check` was clean. No +product test, typecheck, lint, build, Electron, or E2E command was run against +that dirty state. + +## The intended outcome + +For a pane whose old SSH terminal cannot currently be reached, the UI offers: + +- **Try again:** attempt to reattach the exact existing shell without claiming + it exited or spawning a replacement. +- **Start a new terminal:** keep the same pane and leaf, create exactly one + genuinely blank shell, move durable and runtime pane ownership to it, and + leave the unproven old process alive but unbound. + +The second action must never: + +- silently adopt or resume the old shell; +- resume an agent, command, launch token, startup sequence, or saved telemetry; +- create another pane or tab; +- kill a process whose exact incarnation is not proven; +- let a reconnect restore the predecessor after replacement wins; +- let stale renderer work clear, kill, or republish the successor; +- expose provider output before the replacement commits; or +- add replacement intent to SSH, relay, provider, paired-runtime, or stream + wire contracts. + +## What was done during the session + +The branch has 109 first-parent commits over its current merge base, including +two large updates from `origin/main`. The useful progression is below; use the +git history for the complete commit-by-commit record. + +### 1. Reconnect and persistence containment + +The early work stopped SSH reconnect from treating persisted rows as permission +to create UI, made pane binding authoritative during duplicate arbitration, +and added rollback around durable lease retirement. + +Representative commits: + +- `5d8bb18fd3f` — stop reconnect from grafting panes and stacking leases. +- `ae134995c13` — heal duplicate pane leases from older state. +- `ccb082740fc` — let the durable pane binding outrank recency. +- `8d9aeb55e03` — roll back lease retirement when durable writing fails. +- `03d794a0f90` — pass `mayCreate: false` from the production reattach writer. +- `a7558a837ca`, `13ca7630116`, and `8ec025e4e46` — add and strengthen pane, + process, binding, and lease census oracles. + +### 2. Unknown is not death + +The session removed several paths that inferred a dead process from missing, +unavailable, timed-out, or disconnected state. + +Representative commits: + +- `eec807d7594` — stop respawning a shell that may still be running. +- `356d6b52b24` — apply positive-proof requirements to both reattach paths. +- `b9d58423121` and `7589df8b348` — let PTY liveness remain unknown. +- `7cd7fef927f` — a reattach not-found is not itself proof of shell death. +- `e39514e9ab2` and `41613b5c84c` — distinguish bare not-found from a relay- + proven exit and retire a lease only in the latter case. + +This work also stopped one exhausted delivery recovery from tearing down every +session on an SSH host (`07e382ec196`) and made parked delivery recoveries +expire/re-enter correctly (`124e00e8a83`, `601604d17b1`). + +### 3. Exact operation and incarnation fencing + +The branch added or strengthened stale-operation refusal, host-attested shell +identity, remembered-exit identity, and reconnect fences. + +Representative commits: + +- `0de5994be91` and `980e5d846b5` — fence stale writes, resize, and signals. +- `235e45ad0f2` — record host-attested shell identity on a lease. +- `30441f1ebbb` — fence recycled relay PTY IDs by shell identity. +- `8f0c8e3ed35`, `42cbe34523f`, `c14ba64d9dc`, and `0cb187f6035` — require an + exit proof to name the shell for which it grants replacement authority. +- `5793f186e96`, `be02c4e7f5c`, and `901dd06b8d1` — apply incarnation and pane + fences on reconnect, including clients unable to name a shell. +- `c8d08e1b215` — strengthen the incarnation write-fence regression oracle. + +These improvements are valuable, but the round-6 review found that raw PTY ID +is still used as identity in important lease/runtime/cleanup paths. The new +replacement transaction must finish that exact-incarnation work for its own +surface rather than assuming the existing fencing is complete. + +### 4. Pane arbitration and rollback hardening + +Lease arbitration moved from tab-level assumptions toward the actual pane leaf, +and rollback was hardened so an already-expired duplicate cannot be revived. + +Representative commits: + +- `b92498468a5` — arbitrate on the pane, not the tab. +- `1697b3be163` — supersede on the leaf. +- `7724a950ecd` — mutate only the plane that supplied the supersession proof. +- `21ad47c0bc5` — fence duplicate-lease rollback and add a regression oracle. + +The existing `upsertSshRemotePtyLease` key remains `(targetId, ptyId)`. That is +insufficient when an old and a new process reuse the same raw ID but have +different incarnations. Replacement requires exact lease identity. + +### 5. Unreachable-pane product behavior + +The renderer now presents a provider-neutral disconnected card with two +explicit actions and copy that does not claim an unobserved exit. + +Representative commits: + +- `5e1b57bb2f9` — add the two-action disconnected pane affordance. +- `d27927b84b5` — remove a remote-host-only assumption from its copy. +- `037ee535b8a` — centralize unreachable-pane guards. +- `f7eb6f26656` — suppress the entire saved startup set for a fresh shell. +- `b46a1f8773e` — avoid stranding a local pane after restore failure. +- `ab6ce176f3f` — do not unbind a live shell that generic spawn adopted. +- `df6edfcebd2` — center the recovery actions in narrow panes. + +The last item in that list makes the pushed behavior internally safer but also +exposes the unresolved product bug: generic spawn can adopt the incumbent, so +“Start a new terminal” is not guaranteed to start anything new. + +### 6. Tests, design records, and platform evidence + +The session added substantial focused and E2E coverage for local restart, +daemon/WSL survival, SSH pane cardinality, two-host isolation, `MaxSessions=1`, +mixed-version decoding, lease termination, relay exit proof, and renderer +recovery behavior. The main records are: + +- `docs/reference/terminal-session-behavior-contract.md` +- this folder's `goalposts.md`, `design-final-detail.md`, and `resume-plan.md` +- `src/main/ipc/pty.test.ts` +- `src/main/ipc/pty-superseded-operation-fence.test.ts` +- `src/main/persistence.test.ts` +- `src/main/persistence-ssh-lease-termination.test.ts` +- `src/main/runtime/orca-runtime.test.ts` +- `src/main/ssh-reattach-pane-cardinality.test.ts` +- `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts` +- `src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts` +- `src/renderer/src/components/terminal-pane/pty-connection.test.ts` +- `src/renderer/src/components/terminal-pane/pty-transport.test.ts` +- `src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx` +- `tests/e2e/ssh-disconnected-pane-affordance.spec.ts` + +Historical receipts in `goalposts.md` include discriminating native macOS, +Linux, Windows, physical WSL, daemon, and Docker work at earlier candidate +SHAs. They are historical evidence, not a current-HEAD validation receipt. + +Important proof corrections were also recorded during the session: + +- a guard tested only through a direct Store call can still have no production + caller; +- a green E2E that also passes without the fix is a forward guard, not proof; +- the disconnected-pane state is not currently inducible with the existing + Docker host faults; and +- the first recovery grant and a later pane-binding fold were deleted after + review showed their premises were false or their path was unreachable. + +## The proven current bug + +At pushed HEAD, the renderer's disconnected action calls `startFreshSpawn`, +which calls the normal transport `connect`, which invokes generic `pty:spawn`. +The old durable binding is still present. Generic spawn resolves stable pane +ownership and may attach/adopt that incumbent rather than create a blank shell. + +Consequences: + +- “Start a new terminal” can reopen the old terminal. +- A same-ID/new-incarnation result is not representable in the current renderer + result contract. +- The action's success path may compare or clear ownership by raw PTY ID. +- The full product oracle is not running. + +The decisive E2E is +`tests/e2e/ssh-disconnected-pane-affordance.spec.ts`, test “starting a new +terminal adds exactly one shell and leaves the panes alone,” beginning near line 314. It is still unconditionally held by +`test.fixme(true, UNREACHABLE_PANE_INDUCTION_UNAVAILABLE)` near line 318. + +## Dirty WIP that must be removed + +The six uncommitted files implement the rejected +`freshSpawnForUnreachablePtyId`/grant-map design: + +| File | Uncommitted behavior | +| ------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `src/main/ipc/pty.ts` | Adds a max-256 process-global grant map, records a raw `StablePaneOwner` after attach failure, accepts replacement intent on generic spawn, skips adoption, and performs best-effort raw-ID cleanup. | +| `src/preload/api-types.ts` | Adds `freshSpawnForUnreachablePtyId` to generic spawn. | +| `src/preload/index.ts` | Forwards that generic spawn option. | +| `src/renderer/src/components/terminal-pane/pty-connection.ts` | Passes the old `sessionId` as replacement authority. | +| `src/renderer/src/components/terminal-pane/pty-transport-types.ts` | Adds the option to the generic transport contract. | +| `src/renderer/src/components/terminal-pane/pty-transport.ts` | Forwards it to generic spawn and suppresses telemetry with saved startup. | + +Why it was rejected: + +1. Raw PTY ID is not process identity; same-ID/different-incarnation leases and + runtime owners collapse. +2. Renderer input becomes replacement authority on a broad generic IPC. +3. Grants have no TTL, sender binding, one-shot operation state, reload + invalidation, or post-spawn revalidation. +4. Concurrent pane reservations can return another operation's result rather + than re-resolving the caller's intent. +5. Binding and lease changes flush separately, so no atomic replacement exists. +6. Provider output can reach the old model before persistence/runtime commit. +7. Generic spawn still accepts renderer-derived environment and other startup + data, so it is not a guaranteed blank shell. +8. Runtime pane ownership is registered, not exactly transferred; lookup can + continue returning the old record. +9. Renderer callbacks remain authoritative after dispose or a newer attempt. +10. Failure cleanup calls ID-only shutdown and can kill the wrong incarnation. +11. The grant is minted on only one attach-failure path, while the UI action can + be published from other paths, leaving a visible dead action. +12. Replacement intent leaks into `PtySpawnOptions`-adjacent contracts, making + remote-wire and mixed-version reasoning unnecessarily dangerous. + +No uncommitted tests cover this WIP, and no test was run against it. + +Remove only these WIP edits with `apply_patch`. Do not use `git reset`, +`git checkout --`, or another broad restoration command in this dirty worktree. + +## Round-6 independent review result + +Three read-only reviews converged: + +- **Relay/wire review:** request changes. It identified raw-ID identity, + ambiguous cleanup, missing atomic Store commit, precommit output publication, + broad IPC authority, incomplete blank-shell enforcement, and wire- + compatibility blockers. +- **Targeted transaction review:** recommended a dedicated IPC, owner-keyed + mutex, token-keyed idempotency, combined Store transaction, reconnect CAS, + post-image inspection on flush error, and deterministic race tests. +- **Final readiness audit:** request changes. It independently found the same + persistence/runtime/grant/incarnation/renderer/cleanup defects and one hard + feasibility boundary: existing SSH shutdown is raw-ID-only. + +All three reviews were read-only. They ran no tests or builds and changed no +files. + +## Approved bounded redesign + +### 1. Main-issued replacement ticket + +When main classifies an attach as unreachable but not proven dead, return a +typed local result containing an opaque ticket. The ticket is authority, not a +renderer-supplied PTY ID. + +Bind it to: + +- the issuing `webContents`/sender; +- connection and provider generation; +- worktree or folder-workspace identity; +- tab, leaf, and pane key; +- exact old application PTY ID and provider-native ID; +- persisted and runtime incarnation IDs; +- exact active lease identity; +- issuance generation and short expiry; +- a random operation ID; and +- state `issued | running | committed | rejected | unresolved` plus the + idempotent result where applicable. + +Invalidate tickets on renderer reload/destruction, provider replacement, +connection re-registration, expiry, or exact-owner change. A consumed or +rejected ticket must not become valid again after any request was dispatched. + +### 2. Dedicated local IPC + +Add a local-only API such as: + +```ts +pty.replaceUnreachablePane({ ticketId, cols, rows }) +``` + +It should return a structured result such as: + +```ts +{ + ;(ptyId, incarnationId, operationId) +} +``` + +It must not route through generic `pty:spawn` or `PtyTransport.connect`, and it +must not add fields to provider, SSH, relay, paired-runtime, or stream payloads. +Authenticate the sender as the main renderer window before ticket lookup. + +### 3. Shared owner lock and idempotency + +Use one host/worktree/pane owner mutex for: + +- ordinary pane spawn/adoption; +- unreachable-pane replacement; and +- relay reconnect's final owner activation. + +Each waiter must re-resolve its own intent after acquiring the lock. It may not +return another operation's result merely because it waited on the same lock. + +Use the ticket's operation ID for idempotency. A double-click, lost response, +or retry while running awaits the same operation and returns the same result. +It must create at most one shell. + +### 4. Main-constructed blank spawn + +Under the owner lock, re-read the workspace and re-derive CWD. Repeat folder- +workspace path validation at that moment; do not trust the path captured when +the ticket was issued. + +Construct provider spawn options from a narrow main-owned allowlist. Exclude: + +- command and command delivery; +- `sessionId`, `attachOnly`, or resume metadata; +- launch config, token, agent, or telemetry; +- arbitrary renderer environment or environment deletion; +- shell override; +- startup command delivery; and +- sequenced-startup environment. + +Only the validated pane/workspace identity, dimensions, and provider-required +blank-shell metadata may cross the provider boundary. + +### 5. Exact incarnation lease identity + +Lease identity for this transaction must include incarnation, not only +`(targetId, ptyId)`. Old `pty-1/inc-A` and new `pty-1/inc-B` must coexist as: + +- old exact lease: expired/unbound; and +- new exact lease: attached/bound. + +Every mark, remove, attach, rollback, and reconnect CAS touched by replacement +must name the exact lease incarnation. Preserve compatibility with persisted +legacy rows that lack optional incarnation data; unknown identity may not +authorize destructive cleanup. + +### 6. One Store transaction + +Add one Store method, for example `replaceUnreachableSshPaneOwner`, that under +one mutation and one flush: + +1. exact-CASes the old durable pane binding; +2. exact-CASes the old active lease; +3. installs the new binding and incarnation; +4. installs the new exact attached lease; +5. expires the exact old lease without shutting down its process; +6. advances topology and lease mutation versions, including affected already- + expired siblings; and +7. flushes once. + +Snapshot every mutated in-memory field before starting. + +Because durable-file replacement renames the new image before directory fsync, +a thrown flush is not proof that disk stayed old. Inspect the persisted post- +image before deciding what happened: + +- **Old image:** restore memory and clean only a provably exact new process. +- **New image:** treat the transaction as committed. +- **Mixed or unreadable:** kill neither process; mark the operation unresolved, + quarantine publication, and reconcile from disk/provider evidence. + +### 7. Runtime ownership replacement + +Add a non-exit runtime primitive along these lines: + +```ts +runtime.replaceTerminalPaneOwner({ + paneKey, + worktreeId, + expected: { ptyId, incarnationId }, + replacement: { ptyId, incarnationId } +}) +``` + +It must: + +- exact-CAS the current owner; +- clear the old record's pane ownership without `onPtyExit`; +- leave the old process live and unbound; +- move the pane/leaf/handle lookup to the replacement; +- invalidate old handles so they cannot silently target the successor; and +- correctly reset ID-keyed state when raw PTY ID is reused with a new + incarnation. + +Prepare and validate runtime replacement before the Store commit. Make the +final in-memory step synchronous and designed not to fail. If an unexpected +post-durable failure still occurs, never roll durable ownership back to the old +process and never kill either shell; quarantine and retry publication/ +reconciliation idempotently. + +### 8. Quarantine provider output + +Capture the provider object and generation before spawn and recheck them after +every await. Quarantine output for the exact +`{ provider generation, ptyId, incarnationId }` until durable and runtime +replacement commit. + +Only then activate source delivery and publish the new runtime owner. A new +shell reusing the old raw ID must never send bytes into the predecessor model. + +### 9. Fence relay reconnect + +Relay reconnect must acquire the same owner lock after network attach and, +before changing ownership or delivery, exact-CAS: + +- the current pane binding; +- the lease incarnation; and +- the provider/connection generation. + +Only a winning reconnect may set ownership, restore incarnation, activate +source delivery, or register runtime state. A reconnect for the predecessor +must be unable to publish it after replacement commits. + +### 10. Renderer adoption rules + +The renderer calls only the dedicated API. Guard every completion/failure by: + +- connection object not disposed; +- current transport identity; +- current action generation; and +- matching ticket/operation generation. + +Accept same-raw-ID/new-incarnation as a real successor. Adopt only the exact +structured committed result. A stale or disposed callback does nothing: it does +not clear a binding, republish the card, or kill a result. Main owns the durable +transition, so the renderer must never clear the old durable binding after +success. + +## Hard cleanup boundary and selected policy + +Current provider and SSH relay shutdown APIs identify only a PTY ID. If spawn +returns `pty-1/inc-B`, then reconnect races and `pty-1` may refer to a different +incarnation by the time rollback runs. Calling `shutdown('pty-1')` is not exact +cleanup. + +The safe policy for this PR is: + +- do not add replacement intent to the remote wire; +- use existing capability-gated create-operation replay and host-attested + incarnation when they provide conclusive identity; +- clean a failed new shell only when its exact incarnation is provable through + a provider-owned rollback handle or equivalent existing evidence; +- otherwise park/quarantine the ambiguous shell for reconciliation; +- never blind-retry an ambiguous legacy spawn; and +- never risk killing the incumbent merely to avoid a leak. + +Legacy relays without create-operation replay or host-attested incarnation must +fail closed or leave the outcome unresolved. If implementation cannot satisfy +this with current local/provider contracts, stop at the design gate. Do not +silently add `expectedIncarnationId` to remote shutdown; that is a separate, +capability-negotiated wire design and review. + +## Transaction outcome table + +| Outcome | Durable owner | Process action | Publication | +| -------------------------------------------- | ----------------- | ------------------------------------------------- | ------------------------------------- | +| Ticket/pre-spawn validation or fence failure | Old | None | Reject | +| Spawn failure before dispatch | Old | None | Reject; ticket may be safely terminal | +| Spawn response ambiguous after dispatch | Old or unresolved | No blind retry or ID-only shutdown | Park/reconcile | +| Known new shell, post-spawn CAS loss | Winner or old | Stop only the exact new incarnation when provable | Publish nothing | +| Store commit succeeds | New | Leave old process alive; expire exact old lease | Publish new | +| Flush throws; disk has old image | Old | Clean exact new only when provable | Reject | +| Flush throws; disk has new image | New | Leave both processes | Publish new | +| Flush throws; disk mixed/unreadable | Unresolved | Kill neither | Quarantine/reconcile | +| Post-commit runtime/publication failure | New | Never roll back or kill | Retry publication idempotently | + +## Required deterministic oracles + +These are completion requirements, not optional follow-ups. + +### Main transaction + +- Seed an exact failed owner, ticket, binding, and lease. +- One explicit replacement call makes exactly one provider fresh spawn. +- Provider options contain no session, attach, command, startup, resume, + renderer env, launch metadata, or telemetry. +- The same pane remains; the old shell receives no shutdown or synthetic exit. +- Exactly one combined Store handoff occurs and runtime lookup resolves the new + owner. + +### Authorization and lifecycle + +- Missing, expired, replayed, wrong-sender, wrong-host, wrong-worktree, + wrong-pane, and renderer-reload tickets reject before provider spawn. +- Changed/missing binding, persisted/runtime incarnation, lease, provider, or + generation rejects before mutation. +- Folder-workspace path changes after ticket issuance are revalidated under the + lock. + +### Identity and races + +- Old and new may both be raw `pty-1` with distinct incarnations. +- Double-click and lost-response retry create one operation and one shell. +- Ordinary spawn racing replacement does not inherit the replacement result. +- Reconnect before spawn yields no spawn. +- Reconnect after spawn cannot republish the predecessor. +- Provider replacement during every await prevents an unsafe commit. +- A delayed old-incarnation exit cannot affect the new incarnation. + +### Persistence and durability + +- Success survives save/reload with exactly one flush. +- Old exact lease is expired, new exact lease attached, and new binding exact. +- Pre-rename failure restores old memory/disk state. +- Post-rename failure recognizes the new image as committed. +- Mixed/unreadable post-image kills neither process. +- Duplicate-retirement rollback cannot revive an already-expired sibling. +- Same raw PTY ID with different incarnations remains representable. + +### Runtime and renderer + +- Exact non-exit runtime rebind moves pane/leaf/handle lookup to the successor. +- The old owner remains live but unbound and receives no exit. +- Disposed, stale-generation, late-success, and late-failure callbacks cannot + spawn, clear, kill, or republish over newer state. +- Same-ID/new-incarnation structured results are adopted. +- A disposed transport never kills a main-committed replacement. + +### Compatibility and E2E + +- No replacement field appears in provider, SSH, relay, runtime RPC, paired + RPC, or stream payloads. +- Legacy relay degradation is fail-closed/unresolved, never destructive. +- Current-to-release and release-to-current terminal compatibility suites stay + green. +- Unfix the decisive Docker test and add a deterministic production seam that + leaves the target connected while one pane attach is unreachable. +- The E2E must prove old PID plus start ticks stay alive, exactly one blank + shell is added, pane/leaf count is unchanged, new durable `(id, +incarnation)` is bound, new lease is attached, old exact lease is expired, + and no agent/resume/startup transcript is duplicated. + +## Remaining work, in dependency order + +### Phase 0 — resume safely and freeze the invariants + +1. Read this document and the repository `AGENTS.md` instructions. +2. Run the checkpoint commands below. +3. Confirm the PR ref remains at the recorded SHA and inspect any newly arrived + changes before editing. +4. Treat ambiguous cleanup as unresolved rather than adding a remote wire field. +5. If that constraint becomes infeasible, stop and return to the user instead + of widening scope. + +### Phase 1 — remove the rejected WIP + +Use `apply_patch` to remove only the current six-file +`freshSpawnForUnreachablePtyId` and grant-map edits. Preserve every committed +change and any unrelated user work. Run `git diff --check` and verify the only +remaining change is this handoff document before beginning the replacement. + +### Phase 2 — build the identity and persistence foundation + +1. Add incarnation-aware exact lease identity with legacy-row compatibility. +2. Add exact lease lookup/mark/remove APIs required by replacement and + reconnect. +3. Add the single Store replacement transaction and one-flush rollback state. +4. Add post-image inspection for pre-rename, post-rename, and mixed/unreadable + outcomes. +5. Write persistence tests first and demonstrate the intended failure on the + current implementation. + +### Phase 3 — build runtime and coordination primitives + +1. Add the exact non-exit runtime pane-owner replacement primitive. +2. Add the host/worktree/pane owner mutex shared by normal spawn, + replacement, and reconnect final activation. +3. Add operation-token idempotency independent from that lock. +4. Update reconnect to revalidate exact binding, lease incarnation, and + provider generation after it acquires the lock. +5. Add same-ID/new-incarnation and delayed-old-exit tests. + +### Phase 4 — implement the main replacement transaction + +1. Issue typed, sender-bound, expiring, one-shot tickets from every renderer + path that can publish the unreachable action. +2. Add the dedicated preload/main IPC and main-window sender check. +3. Re-derive CWD and validate folder workspaces under the lock. +4. Construct the blank provider spawn from the main allowlist. +5. Capture provider generation and quarantine exact new output. +6. Revalidate after every await; prepare runtime; execute the Store commit; + finish runtime activation; publish output/result. +7. Implement all transition-table outcomes without ID-only cleanup. + +### Phase 5 — adopt through the renderer + +1. Remove replacement from generic transport/spawn types. +2. Thread the typed ticket and structured committed result through the + disconnected action only. +3. Add disposed/current-transport/action-generation guards. +4. Remove renderer-side binding clearing or committed-result killing. +5. Verify UI layout, copy, and behavior through Electron Playwright CDP as + required by repository instructions. + +### Phase 6 — validate and review once + +1. Run the focused deterministic suite. +2. Make the Docker E2E inducible, remove its unconditional `fixme`, and prove + it fails for the intended reason on the unfixed candidate before claiming + it as evidence. +3. Run typecheck, lint, full unit tests, build, reliability gates, and relevant + SSH/daemon/cross-version regressions. +4. Run independent repository and release-readiness reviews, including + security, performance, mobile/backcompat, Windows, Linux/glibc, SSH, folder + workspace, and provider-neutrality coverage. +5. Resolve every P0/P1 and relevant P2. If another architectural invariant + fails, pause instead of entering an unbounded seventh patch loop. + +### Phase 7 — deliver + +Only after the implementation and reviews are clean: + +1. Commit the replacement work. +2. Push explicitly to + `origin/nwparker/sta-3077-d-affordance-and-review`. +3. Update PR #13326's body with the transaction, compatibility behavior, and + exact validation receipts. +4. Resolve the seven saved review threads only when their findings are actually + addressed. +5. Request `@coderabbitai review` and monitor CI to completion. +6. Use the `orca-linear` workflow to post one concise STA-3077 completion + update and attach the PR if needed. + +## First safe resume commands + +Run from the primary worktree before editing: + +```bash +git status --short --branch --untracked-files=all +git rev-parse HEAD +git rev-parse origin/nwparker/sta-3077-d-affordance-and-review +git rev-parse origin/main +git rev-list --left-right --count origin/main...HEAD +git diff --stat +git diff --check +``` + +Expected before any external state changes: + +- HEAD and the PR ref are both `df6edfcebd2ea401a6feda1fccf31189e57eceda`; +- the six rejected production-file edits are present; +- this handoff file is the only additional local artifact; and +- `git diff --check` exits 0. + +Do not use the configured upstream to infer PR state. + +## Suggested validation commands after implementation + +Adjust the focused file list only when the final implementation clearly moves +the covered behavior. Record exact commands and outcomes in the PR. + +```bash +pnpm exec vitest run --config config/vitest.config.ts \ + src/main/ipc/pty.test.ts \ + src/main/persistence.test.ts \ + src/main/persistence-ssh-lease-termination.test.ts \ + src/main/runtime/orca-runtime.test.ts \ + src/main/ssh-reattach-pane-cardinality.test.ts \ + src/main/ssh/ssh-relay-session.test.ts \ + src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts \ + src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts \ + src/renderer/src/components/terminal-pane/pty-connection.test.ts \ + src/renderer/src/components/terminal-pane/pty-transport.test.ts \ + src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx \ + --reporter=dot + +pnpm typecheck +pnpm lint +pnpm test +pnpm build +``` + +For the Docker product oracle, use the repo's Electron Playwright configuration +with `ORCA_E2E_SSH_DOCKER=1` after adding the deterministic induction seam and +removing the test's unconditional `fixme`. Keep the run POSIX-only as the +existing Docker fault harness requires. + +Run the current↔release cross-version terminal suite and the relevant Docker +SSH cardinality, `MaxSessions=1`, and two-host scripts as part of broad +regression coverage. + +## Validation record at pause + +### Current redesign/WIP + +No product validation was run after the rejected six-file WIP was introduced. +No product validation was run during the three final read-only audits or while +writing this handoff. Therefore this document claims no current test, typecheck, +lint, build, Electron, Docker, cross-version, or packaging pass. + +### Read-only checkpoint commands run for this handoff + +The following repository inspections completed successfully: + +```bash +git status --short --branch +git status --short --branch --untracked-files=all +git rev-parse HEAD +git branch --show-current +git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' +git rev-parse origin/nwparker/sta-3077-d-affordance-and-review +git rev-parse origin/nwparker/sta-3077-reattach-pane-cardinality +git rev-parse origin/main +git rev-list --left-right --count origin/main...HEAD +git merge-base origin/main HEAD +git log --oneline --decorate -12 +git reflog --date=iso +git diff --stat +git diff --name-status +git diff --check +``` + +Observed results are recorded in the repository checkpoint above. In +particular, `git diff --check` exited 0. + +Historical validation details are preserved in `goalposts.md` and +`resume-plan.md`. Those receipts belong to their recorded SHAs and must not be +reported as validation of the final replacement candidate without rerunning +them. + +## Security, compatibility, migration, and release risks + +- **IPC authority:** a renderer-supplied PTY ID is forgeable/piggybackable; + tickets must be opaque, sender-bound, short-lived, one-shot, and revalidated. +- **Same-ID reuse:** raw PTY ID cannot distinguish an incumbent from a + successor. Every destructive or ownership operation needs exact incarnation. +- **Durability:** a thrown fsync after rename can leave the new image on disk. + Blind in-memory rollback can diverge from restart state. +- **Remote ambiguity:** ID-only shutdown cannot safely clean same-ID reuse. + Prefer a visible/reconcilable leak to killing an unproven process. +- **Output isolation:** precommit bytes from a successor can corrupt the old + pane model unless quarantined by provider generation plus incarnation. +- **Mixed versions:** clients and hosts update independently. Keep replacement + local; any future exact-shutdown wire field needs capability negotiation and + both-direction live skew tests. +- **Legacy persisted rows:** missing incarnation is unknown, not permission to + mutate or delete. Migration must be additive and old-reader-safe. +- **Folder and SSH workspaces:** re-resolve current paths and host generation + under the lock; neither git worktree presence nor local filesystem access may + be assumed. +- **Cross-platform:** preserve macOS, Linux/glibc floor, Windows, WSL, path, and + shortcut rules even though the Docker fault harness is POSIX-only. +- **Performance:** avoid global scans, unbounded ticket/operation maps, + repeated fsyncs, or hot-path lock contention. Bound retention and measure + affected reconnect/spawn paths. +- **Release evidence:** the key E2E remains held. The change is not release- + ready until that product path is deterministic and discriminating. + +## Stop conditions + +Pause and return to the user if any of these occurs: + +- exact cleanup appears to require an unreviewed remote-wire change; +- the Store/runtime transaction cannot distinguish old, new, and mixed durable + post-images; +- replacement requires treating unknown identity as proof; +- the deterministic E2E still cannot enter the actual production state; +- the shared lock introduces a new global serialization or deadlock risk; or +- the next independent review finds another architectural invariant failure. + +The point of this handoff is to prevent another opaque loop: one bounded +redesign, deterministic evidence, one adversarial review, then either delivery +or a principled stop. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/README.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/README.md new file mode 100644 index 00000000000..2dab127658c --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/README.md @@ -0,0 +1,265 @@ +# Terminal session correctness and authority handoff + +> **Status: preserved and paused, not complete, not release-ready.** +> +> The comprehensive program is at **0/8 proven goalposts** and +> **2/13 proven journeys**. No agent may narrow, replace, or mark a goalpost +> complete without the user's explicit approval and the proof named in this +> folder. + +This folder is the sole continuation entrypoint for the work previously called +“React 185” or “terminal session authority.” That shorthand is too narrow: the +program covers terminal identity, ownership, exact operations, reconnect, +delivery, migration, daemon and SSH lifecycle, paired and remote runtimes, +cross-platform compatibility, performance, and removal of superseded code. + +## Read in this order + +1. Read all three preserved source documents with the exact `git show` commands + under [Repository checkpoint](#repository-checkpoint). The normative design, + delivery ledger, and original pause handoff are mandatory, not optional + implementation detail. +2. [`goalposts.md`](./goalposts.md) — the current completion contract, amendable + only by an explicit user decision. +3. [`related-open-work.md`](./related-open-work.md) — live PR/issue overlap and + conflicts. +4. [`resume-plan.md`](./resume-plan.md) — safe continuation order and required + validation receipts. + +The older `terminal-session-authority-handoff-2.md` is useful as an adversarial +report, but it is not authoritative. In particular, its “descoped,” “shipped,” +identity-equivalence, timer, and LOC claims must not be copied as settled facts. + +## Non-negotiable user objective + +Deliver one coherent long-term terminal-session correctness model that: + +- removes the root causes instead of leaving quarantine, retry-window, + sliding-window, optional-metadata repair, or other reconciliation as the + final correctness mechanism; +- never treats missing, unavailable, timed-out, disconnected, or unknown state + as proof that a process is dead or that replacement/destruction is safe; +- works for local terminals, daemon, WSL, direct and nested SSH, paired runtime, + remote server, folder workspaces, floating workspaces, and git worktrees; +- remains correct when clients and hosts update independently; +- works on macOS, Linux at the supported glibc floor, Windows, WSL, and Docker + OpenSSH; +- has no correctness, security, latency, throughput, memory, restore, startup, + backpressure, or large-pane regression; +- finishes with **strictly less aggregate production code than the frozen + pre-program baseline** after all superseded implementations are deleted; and +- is proven through real production paths and discriminating end-to-end + journeys, not inferred from test counts or green CI. + +Correctness and performance cannot be weakened to reach the code-size target. +The code-size target also cannot be waived because a replacement architecture +was added beside the old one. + +## Authority and scope rules + +- Direct user instructions govern scope. +- The original normative design remains the contract except where a later, + explicit user decision changes a named requirement. +- This handoff tightens G6 to **strictly net-negative production LOC** because + the user explicitly asked to end with less production code than the original + baseline. +- The default frozen accounting baseline is + `5ed45739e94bdf6460364e033bfcec9b32c0b42a`, the base recorded by GitHub for + PR #12600. This program subsumes that containment PR. Every later + program-attributable merged prerequisite and every stack layer must be counted + in aggregate; rebasing may not roll the accounting baseline forward and hide + earlier additions. Changing this baseline requires an explicit user decision. +- Focused tests, green CI, a review comment, a behavior-contract document, an + open PR, or a construction milestone cannot amend the design or promote a + release goalpost. +- A replacement design is allowed, but it must map every original invariant, + risk, and journey to an equal or stronger mechanism, close the known liveness + gap, and receive explicit user approval before the ledger changes. +- Only `not started`, `partial`, and `proven` are valid gate statuses. + “Descoped,” “superseded,” and “met for what shipped” are not valid statuses + for G0–G7 under the current objective. +- A narrow incident PR may be independently useful or mergeable. It does not + reduce the comprehensive denominator of eight goalposts and thirteen + journeys. + +## Settled status + +| Scope | Status | Meaning | +| -------------------------- | --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Construction preservation | **Complete locally; not remotely durable** | The 814-path construction tree and the original three authority documents are preserved at local branch `nwparker/react185-authority-snapshot` / `fddb19f6977`. | +| Comprehensive goalposts | **0/8 proven** | G0–G4 and G6 are partial. G5 and G7 are not started. | +| Required journeys | **2/13 proven** | Journeys 1 and 2 have clause-selective discriminating oracles run on every environment they name. The other eleven remain unproven. | +| Current branch / PR #13111 | **Implemented, open, unapproved, incomplete** | Useful narrow SSH containment; not a long-term completion and not merged. | +| PR #13110 | **Open, independent, unapproved** | Useful preload artifact guard; not shipped and has an unresolved major review thread. | +| Less-code gate | **Failed on current candidate** | PR #13111 is net positive; the preserved construction snapshot is net `+60,903` production LOC. | + +The numeric release checkpoint is therefore: + +- **G0:** partial +- **G1:** partial +- **G2:** partial +- **G3:** partial +- **G4:** partial +- **G5:** not started +- **G6:** partial and currently failing its final LOC condition +- **G7:** not started +- **Total:** **0/8 proven, 6/8 partial, 2/8 not started** +- **Journeys:** **2/13 proven** (Journey 1 natively on macOS/Linux/Windows; Journey 2 on macOS/Linux/physical WSL) + +This is the **program artifact ledger**. Its partial G0, G1, G3, and G4 evidence +exists only in the preserved construction snapshot and historical receipts. It +does not mean those implementations exist on the current #13111 branch. The +current candidate contributes credible partial G2 evidence and limited +compatibility/review evidence, but it proves **zero** complete release gates. + +## Repository checkpoint + +Verified before creating this documentation folder: + +- Worktree: + `/Users/nwparker/orca/workspaces/orca/eye-React-185` +- Current branch: `nwparker/sta-3077-reattach-pane-cardinality` +- Current HEAD: `5369479be2953f45cea9ab5cfcde756dd9660548` +- HEAD subject: `fix(pty): let liveness say unknown instead of forcing it to say dead` +- Local `origin/main`: `0d29497f8279e2d4c2d26ffc8f3fb68cee2756a7` +- Merge base: `e6e197feeddd1adda066c5cd22f7ee056a12c8c1` +- Distance from local `origin/main`: 15 commits ahead, 19 behind +- The worktree was clean before this handoff. The expected handoff-only changes + are this folder and its `.gitignore` allow-list entry. +- Preserved construction branch: + `nwparker/react185-authority-snapshot` +- Preserved construction commit: + `fddb19f6977ae4ba4764c32afc9fe104c1ed2549` +- Preservation location: local ref only; the branch has no upstream, tag, or + remote branch containing this commit +- Snapshot status: **unshippable salvage/reference only** + +The exact commit ID is the preservation anchor. Do not checkout, switch, reset, +rebase, commit, or push from the snapshot branch in the primary worktree. Read +its authority documents without switching branches: + +```bash +git show fddb19f6977:docs/reference/terminal-session-authority.md +git show fddb19f6977:docs/reference/terminal-session-authority-delivery.md +git show fddb19f6977:docs/reference/terminal-session-authority-handoff.md +``` + +Mine tests, design arguments, or narrow implementations only through `git show` +or a disposable isolated worktree, and only after verifying that they address a +reachable production path on the current base. A local tag protects against +branch movement and ordinary GC only; it does not survive loss of this machine. +Cross-machine/crash preservation requires a copy-verified off-machine bundle or +an explicitly authorized remote ref. Do not infer permission to push it. + +## Current narrow candidate: what is real + +At current HEAD / PR +[#13111](https://github.com/stablyai/orca/pull/13111): + +- SSH relay reattach passes `mayCreate: false` to stop that call site from + creating durable pane/layout state. +- Duplicate complete pane leases are superseded and excluded from reattach. +- Existing duplicate complete leases are reconciled during reconnect. +- Reattach failure paths require positive evidence before cold respawn. +- PTY inventory can report `unknown` rather than collapsing an unavailable + provider into `false`/dead. +- Focused tests and GitHub checks are green at HEAD. + +These are credible partial G2 improvements. They do **not** establish the +comprehensive design or release gates. + +## Current narrow candidate: what remains unproved or wrong + +- Lease row identity is still `(targetId, ptyId)`. Pane fields remain optional. + Active-pane cardinality is enforced by sibling expiration and a healing scan, + so the end state remains reconciliation-based rather than structurally + pane-keyed. +- Incomplete legacy leases bypass pane arbitration. +- Superseded remote shells are deliberately left running and unreachable. The + PR body acknowledges that the “accumulates unused shells” half of the incident + remains unresolved. +- The repeated-reconnect Docker test still passes with and without the fix: it + is a forward guard, not causal proof. Its sibling in the same spec — 'leaves a + lease whose durable pane is gone unbound…' — was reported as discriminating, + but that DID NOT REPRODUCE on a second machine: with `mayCreate: false` + removed from the reattach call site and the app rebuilt, both tests still + passed. Its induction races `pty:kill` against a severed transport, so when + the kill lands the lease is cleaned up and there is nothing to graft. Treat + both tests as forward guards. No journey is proven. +- The final Docker settle assertion has an unresolved major review thread: a + late pane or shell can appear after the polling assertion has already passed. +- The production-call-site wiring oracle reads source text. It does not execute + the full production reattach path. +- `mayCreate: false` is wired at one production binding call site, not proven + across every recovery/reattach runtime. `removeSshRemotePtyLease` still has one + production caller, the spawn-persistence rollback. +- `WEDGED_DAEMON_GRACE_RETRIES = 11` remains reachable and still permits retry + count to drive daemon replacement after unknown inventory. +- `terminal-input-quarantine.ts` remains reachable. +- No current-tree proof covers physical Windows, WSL, the Linux glibc floor, + paired runtime, remote server, two independent SSH hosts, live mixed-version + peers, or production-scale performance. +- There is no candidate-versus-baseline performance result for the new + per-upsert sibling scan or reconnect healing pass. +- PR #13111 is open, has no approval decision, and is not shipped. + +Current HEAD census against its merge base, before this documentation change: + +| Category | Additions | Deletions | Net | +| --------------------------------------------------- | --------: | --------: | -------: | +| Production source, including a fixture under `src/` | 374 | 65 | **+309** | +| Tests and end-to-end tests | 944 | 30 | +914 | +| Documentation | 146 | 0 | +146 | +| CI/configuration | 43 | 0 | +43 | +| Total GitHub diff | 1,507 | 95 | +1,412 | + +The production source plus Docker runner is net `+350`. Whichever taxonomy is +used, the categories must remain separate and G6 is not proven. + +## Corrections to the previous handoff + +- PRs #13110 and #13111 are **open and green**, not shipped. +- #13110 is implementation-independent from #13111, but emitted-preload safety + remains inside G0/G5/G7 and any retained #13110 production/build logic counts + in the aggregate G6 census. +- The snapshot genuinely failed to wire `mayCreate`, pane-cardinality + arbitration, or a complete shell-lifecycle fix into the affected legacy SSH + path. +- Pane key, PTY ID, incarnation ID, a narrow CAS, and a three-valued daemon + resolver pre-existed and should be reused where their semantics match. +- Existing `connectionId` and `worktreeId` do **not** establish equivalence to a + final-host-minted authority identity and canonical host-local namespace. +- The normative design never banned all timers. It permits bounded waits and + backoff; it forbids elapsed time or retry count **alone** from deciding + identity, liveness, takeover, replacement, or destruction. +- The design does have a liveness gap: a permanently lost retained consumer can + hold compaction at capacity without a sufficiently explicit bounded operator + recovery path. G0 and G3 cannot become proven until that is resolved. +- The claimed approximately 6,500-line comparison implementation has no + reproducible census and cannot justify deleting requirements. + +## Definition of done + +The comprehensive work is done only when all of the following are true on one +rebased, converged candidate SHA: + +- every G0–G7 section in [`goalposts.md`](./goalposts.md) has its named + production implementation and current proof; +- all thirteen required journeys pass with their full oracles; +- each new causal oracle is red on the unfixed baseline and green on the final + candidate; formal verification may supplement but never replace a required + live journey; +- every open incident bound to the mandatory issue-to-journey matrix in + [`related-open-work.md`](./related-open-work.md) passes its discriminating + oracle or is ruled unrelated with explicit evidence and user acceptance; +- correctness, security, wire, migration, platform, packaging, performance, + restore, and scale reviews have no unresolved P0–P2 findings; +- the aggregate production source census is strictly net-negative against the + frozen pre-program baseline, including all program-attributable prerequisite + PRs and stack layers; +- no superseded reconciliation, quarantine, duplicate state machine, legacy + writer, or test fixture remains reachable in production; and +- the user explicitly accepts any deliberate change to the normative design. + +Anything less must be reported as partial, regardless of test count or PR +state. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/RESUME-PROMPT.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/RESUME-PROMPT.md new file mode 100644 index 00000000000..88cff53f68b --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/RESUME-PROMPT.md @@ -0,0 +1,111 @@ +# Resume prompt — terminal session ownership + +Paste the block below into a fresh session. Everything it needs is in the repo. + +--- + +You are taking over an in-flight implementation in this repo. Read these, in +order, before doing anything: + +1. `docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md` + — the tracked goalposts. This is the contract. Report progress against it. +2. `docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html` + — the design in plain English (open it; it has diagrams). +3. `docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md` + — the full detail: record shape, death rule, reattach algorithm, migration + order, and the oracle list. +4. `docs/reference/terminal-session-correctness-handoff-2026-08-07/counsel-design.html` + — how the design was reviewed, and the defects the review found in earlier + shipped work. Read at least the "What I got wrong" section. + +## The problem being fixed + +Reconnecting to an SSH host multiplied a user's terminals (2 → 19 → 20), and a +coding agent could be resumed twice into one transcript. Root cause is not one +bug: it is that ownership bookkeeping is filed under the wrong key and written to +two places, and that the client infers "the program died" from things that do not +prove it. + +The design in one line: **keep our own notes about which terminal is whose, and +never guess whether a program died.** + +## State + +Branch `nwparker/sta-3077-reattach-pane-cardinality`. Landed and committed: + +- `e2524b0472f` — an identity mismatch is no longer read as death (goalpost S1) +- `c51be8072ba` — pane identity is no longer sent on reattach (S2) +- Goalposts, design docs (S6 verdict recorded) + +**3 of 7 step goalposts proven. Net production lines: −24.** + +Two older PRs (#13110, #13111) are **held, unmerged, deliberately**. The owner +decided: no stopgap, get the design right. Do not merge them without being asked. + +## What to do next + +Three goalposts have **failing oracles already written**, each with the exact +production change identified. This is implementation against a fixed target. + +**S3 — a failed reattach must never fabricate an exit.** +Oracle: `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts` (6 red, 2 green). +In `handlePtyReattachFailure` (`src/main/ssh/ssh-relay-session.ts`), the +not-found case currently sends the pane a synthetic `pty:exit { code: -1 }`, +clears provider state, deletes ownership and expires the lease — all claims about +a process we know nothing about. Collapse it into the non-destructive branch that +already exists a few lines above (`restoreRequired = 'reattachAttemptsExhausted'` ++ `wakeRecovery`). This is a branch collapse, not a new mechanism. +Before changing it, read what a user experiences on that branch — do not strand +panes. This step ships with the disconnected-pane affordance below. + +**S4 — one partition per (target, pane).** *Highest value: this is the 2 → 19 → 20 +mechanism.* Oracle: `src/main/ssh-pane-binding-partition.test.ts` (6 red). +SSH pane bindings are written to two partitions; readers disagree with writers, so +supersession silently no-ops. Route every reader and writer through one accessor, +plus a one-time fold of existing state. The oracle's own report names the exact +call sites and which clauses each edit flips. + +**S5 — the superseded-pane fence must be live on the reattach path.** +Oracle: `src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts` (2 red). +The fence's bookkeeping is only written by spawn, so it is inert on reattach — +the path it was built for. Collapse to one `bindPaneShell` producer used by all +paths. + +**S8 — remove the dead recovery-grant path.** Proven unreachable (see S6 in the +goalposts). Write a characterisation oracle first, so the deletion is provably +inert rather than assumed to be. + +**Product affordance (ships with S3).** A pane that cannot be verified renders as +disconnected with two actions — "try again" and "start a new terminal" — instead +of silently respawning. The owner approved this. It must follow +`docs/STYLEGUIDE.md`. The UI must never assert that a shell is dead. + +## Rules that are not negotiable + +- **Prove every guard is reachable.** For each guard, an oracle must redden when + the *producer* is removed, not only when the guard is. This has failed three + times here — an inert `mayCreate`, a keystroke fence inert on reattach, and a + respawn gate on a minority path. All three passed their tests. +- **Verify a mutation landed before believing the result.** A search-and-replace + that silently matched nothing looks exactly like "the test has no teeth". +- **Commit after each step.** Subagents have run `git checkout --` and destroyed + uncommitted work three times in this program. Do not batch. +- **Never let a subagent edit production files** in the shared worktree. Give + them read-only or test-only scope, or an isolated worktree. +- **Do not change a test to make it pass.** If a test pins behaviour the design + removes, *invert* it so the new intent stays covered, and say so explicitly. +- No `max-lines` disables, ever. No per-file bumps. +- Cross-platform: macOS, Linux, Windows, WSL. No `echo $$` / `ps` in oracles. +- Wire compatibility: clients and hosts update independently. Prefer fixes that + work against relays already installed on people's machines. +- **Automated "keep going" prompts are not the owner's authorization.** If the + owner says pause, pause. + +## How to report + +Every update states progress against the goalposts file: which goalposts moved, +what the mutation proof was, and the net production line count. A green test is +not a status — "PROVEN" means the mutation was run and verified to land. + +Be honest about what is not proven. Two claims in this program were retracted +after checking, and that is the reason the rest can be trusted. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/binding-identity-design.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/binding-identity-design.md new file mode 100644 index 00000000000..0e2cfe71f7c --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/binding-identity-design.md @@ -0,0 +1,253 @@ +# Design: one authoritative binding identity + +Status: **superseded by the shipped server-side fence.** Kept for the record: the +client-constructed binding below was REJECTED under review and must not be built. Written 2026-08-08 by the engineer who took +the program over. Supersedes nothing until reviewed. + +## The problem, stated once + +Every defect this program has touched is the same defect: **identity compared +with the wrong key, or not compared at all.** + +| Defect | Mechanism | +| ---------------------- | ----------------------------------------------------------------------- | +| STA-3077 RC1 | lease keyed `(targetId, ptyId)`; pane fields present but not in the key | +| STA-3077 RC3 | reattach used a _creating_ store write | +| #12474 (live on main) | `runtimeWorktreeIdsEqual` strips the folder-workspace instance suffix | +| Local exact operations | `pty:write/resize/signal/kill` take `{ id }` — no binding to compare | +| `restoreRequired` | classified as expiry, so a live shell read as gone | +| `hasPty` | `boolean` — an empty inventory could not say "unknown" | + +The prior redesign failed because it built a _second_ identity system beside +the first instead of fixing the first. It reached +60,903 production LOC and +fixed none of RC1–RC3. + +## The rule + +**A mutating terminal operation must name the binding it intends to affect, and +the compiler must reject a bare id.** + +Not a new subsystem. A type, one comparison, and a signature change. + +## 1. `PtyBinding` — one branded type + +```ts +declare const bindingBrand: unique symbol + +export type PtyBinding = Readonly<{ + hostId: ExecutionHostId // exists — LOCAL_EXECUTION_HOST_ID | ssh: + worktreeId: WorktreeId // exists — `${repoId}::${path}[::workspace:]` + paneKey: PaneKey // exists — `${tabId}:${leafId}`, already branded + ptyId: string // exists + incarnationId: PtyIncarnationId // exists — already branded +}> & { readonly [bindingBrand]: true } +``` + +Every field already exists and is already persisted. Nothing is invented. + +Construction is the whole point: `PtyBinding` is producible **only** by +`bindingFromAuthority()` — reading the durable store, a spawn result, or an +attach reply. There is no public constructor from loose strings, so a caller +cannot fabricate one, and `as PtyBinding` is banned by lint. + +## 2. Mutating IPC carries the binding + +Today, on the local path: + +```ts +pty: write({ id, data }) +pty: resize({ id, cols, rows }) +pty: signal({ id, signal }) +pty: kill({ id }) +``` + +There is no fence to test because there is nothing to compare. That is a +production gap, not a test gap. + +After: + +```ts +pty: write({ binding, data }) +pty: resize({ binding, cols, rows }) +pty: signal({ binding, signal }) +pty: kill({ binding }) +``` + +The handler resolves the binding against the authoritative record and rejects a +mismatch — the same compare-and-swap `persistPtyBinding` already performs for +`expectedBinding`. A stale renderer cannot reach a successor pane or a reused +PTY id, which is invariants 1–4 of the original design, enforced rather than +asserted. + +**Compatibility:** the ID-only channels stay for one release behind the existing +capability negotiation, since clients and hosts update independently. They are +marked deprecated, are not reachable from authoritative paths, and are deleted +in the release after — that deletion is where the LOC comes back. + +## 3. One comparison, not twenty-six + +```ts +export function bindingsEqual(a: PtyBinding, b: PtyBinding): boolean +export function sameNamespace(a: PtyBinding, b: PtyBinding): boolean +``` + +Delete the hand-rolled comparisons. `runtimeWorktreeIdsEqual` — the #12474 bug — +is one of them; ~26 repeat a host-id/namespace-id comparison inline. Each +hand-rolled copy is a future drift, and #12474 proves drift already happened. + +## 4. Three-valued liveness + +Landed (`5369479be29`). `hasPty: boolean | null`, `null` never authorizes +destruction. + +## What this deletes + +| Target | Est. LOC | +| ----------------------------------------------------------------- | -------: | +| ID-only IPC channels + handlers (release after next) | ~250 | +| Hand-rolled identity comparisons (~26 sites) | ~180 | +| `pty-source-replay-index.ts` (done) | 201 | +| Inference sites that exist only because a binding was unavailable | ~200 | + +Net direction is negative once the deprecated channels go. It is net-positive in +the release that adds the type, and the recorded G6 decision permits that when +justified. + +## What this deliberately does not build + +No durable journal, no per-consumer cumulative cursors, no cryptographic device +principals, no parallel authority service. Verified against mature prior art: +bounded replay plus lifecycle-in-the-attach-reply is what shipping systems use, +and their entire persistent-terminal subsystem is ~6,500 LOC. + +If a reviewer can name a concrete event sequence where bounded replay loses or +double-applies an outcome, that conclusion changes. Nobody has yet. + +## How each claim gets falsified + +| Claim | Oracle | +| ---------------------------------------- | ---------------------------------------------------------------------- | +| A stale op cannot reach a successor pane | Send a captured binding after the pane is recreated; must be refused | +| A stale op cannot reach a reused pty id | Same, after incarnation change | +| Same-path workspaces do not collide | The 5 skipped tests in `workspace-namespace-terminal-identity.test.ts` | +| Reattach never creates | Proven, discriminating (`ed10a467883`) | +| Unknown never destroys | Revert three-valued `hasPty`; daemon spec reddens | + +Every row must fail with its guard removed, verified under an isolated +`TMPDIR` — the e2e harness keys its seeded-repo pointer on a machine-global +tmpdir path, so a shared machine can both fabricate and mask a red. + +--- + +# Post-review record (2026-08-08) + +## The proposal above was rejected, correctly + +Verified against the code, three load-bearing claims in it were false: there is +no `WorktreeId` type; `PtyIncarnationId` is a bare `string` alias, not branded; +and `incarnationId` is optional in the store and never reaches the renderer, so +the side asked to construct the binding cannot. It was also a _second_ identity +comparison beside `resolveStablePaneOwner` (`src/main/ipc/pty.ts:647`), which +already resolves pane ownership against runtime and store and already throws +`terminal_pane_owner_conflict` — with 9 call sites, all on spawn/adopt, none on +a mutating handler. The fence existed; it was not being called. + +## What shipped instead + +`isSupersededPtyId` in `src/main/ipc/pty.ts`, consulted by `pty:write`, +`pty:writeAccepted`, `pty:resize` and `pty:signal`. Main keeps `ptyPaneKey` and +`paneKeyPtyId` in lock-step, so their disagreement is proof the caller's id was +superseded. No wire change, no renderer change, nothing added to the input +payload. `pty:kill` is deliberately exempt: a superseded PTY is orphaned and +reclaiming it is the point. + +## Peer review of that shape + +Four comparable agent IDEs were studied. The verdict was to keep this shape. +None sends a composite identity per keystroke; the one that fences keystrokes +sends a single opaque server-minted scalar. The best peer formulation is _keep +the id as the sole lookup key and make the second field a rejection predicate, +never a key component_ — which is what this does, with the owner read from +authority rather than accepted from the caller. + +## Known gap, stated precisely + +**The fence compares a binding, not an incarnation.** If a pane respawns under +the _same_ ptyId, `paneKeyPtyId.get(paneKey) === ptyId` still holds and a write +composed for the dead shell passes. + +The obvious remedy — mint a fresh ptyId on every spawn — is wrong here. +`ptySessionIdForAgentCreateOperation` (`src/main/daemon/pty-session-id.ts:26`) +is deterministic _by design_ so a replayed agent-create is idempotent rather +than spawning a second shell. Randomising it would trade this narrow gap for a +duplicate-spawn bug, which is worse. + +So the residual exposure is one path: an agent-create PTY dies, the same +operation is replayed, the id is reproduced, and a caller holding a pre-death +reference writes into the successor. Closing it needs the caller to carry the +incarnation — the rejected proposal — or a per-attachment token, which is a wire +change. Neither is justified by evidence today. Recorded rather than hidden. + +## Adopted from the peer review + +- The "no binding recorded" branch is an explicit, commented decision (permit), + not an accident — an omitted guard on an optional field is exactly how one + peer reintroduced this defect class. +- A classifier's default must be non-destructive: unknown resolves to reattach, + and only a positive "session not found" authorises a respawn. + +## Not yet adopted, ranked + +1. Typed end-reason recorded at end time (`detached` vs `terminal-exited`), so a + user quit is not a resume candidate. Aimed squarely at duplicate resume. +2. Compare-and-swap before any _delayed_ destructive write. Three-valued probes + stop us acting on unknown; CAS stops us acting on stale-known. +3. Derive the pane inventory from the authority snapshot rather than merging into + a local store. A projection cannot grow; an accumulator can — which is what + 2 -> 19 -> 20 was. +4. Durable intent-to-kill, since kill is the reclamation path for orphaned leases + and a one-shot renderer broadcast should not be its only chance. + +--- + +# Duplicate agent resume: why I did not add the fix I recommended + +Adoption item 1 was a typed end-reason recorded at end time — `detached` (the +user quit; not a resume candidate) versus `terminal-exited` (it died under the +agent; resumable) — on the grounds that a boolean `ended` cannot express the +difference, which is why systems keep resuming things they should not. + +I went to implement it and stopped, because the codebase already contains that +idea three times over. + +`SleepingAgentSessionRecord` (`src/shared/agent-session-resume.ts`) carries: + +- `origin?: 'worktree-sleep' | 'quit' | 'live'` — added so worktree activation + would not launch a tab that a warm reattach had already restored (#5232); +- `restoreOnTabOpenOnly?: boolean` — added so a mobile wake would not background + mount every slept tab and respawn the workspace the user just slept (#11598); +- `automaticResumeBlockedBy?: 'legacy-orchestration-worker'` — added so a + relaunch could not race a durable orchestration assignment. + +Three fields, three incidents, one defect: something resumed that should not +have. They are consulted at 22 non-test sites. A fourth flag — however well +typed — is the fifth containment cycle, which is the failure pattern this +program exists to break. + +The peer designs that do not have this bug do not have a better flag. They have +a different shape: + +1. **Nothing resumes automatically.** Resume is a button. On click it resumes + into a _brand-new terminal id_, re-points the pane, then kills the old + terminal — so two live agents for one pane never exist even transiently. +2. **Two-agents-in-one-terminal is unrepresentable.** The terminal id is the + agent binding's primary key with a cascade to the session row, and live reads + inner-join on the session being active — so a dead terminal's agent cannot + appear in a read no matter how the terminal died. + +Neither is a 30-line change here, and (1) is a product decision about whether +automatic resume stays a feature. That belongs to the user, not to me. + +What is safe to say: the next duplicate-resume incident should not be answered +with a fourth predicate on this record. The record already proves that approach +does not converge. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/counsel-design.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/counsel-design.html new file mode 100644 index 00000000000..34c224c6352 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/counsel-design.html @@ -0,0 +1,278 @@ + + + + + + Design Counsel — Terminal Session Ownership + + + +
+
+

Design Counsel — Terminal Session Ownership

+

Three review rounds across two models, an adversarial fact-check, and a close-out · the design that emerged is not the one I proposed

+
+ +
+
1
Live bug found
+
2 → 1
Durable records
+
3
Review rounds
+
78 / 12
Facts right / wrong
+
4
Unresolved conflicts
+
~900–1250
Lines deleted
+
+ +
+

The headline

+

The counsel found a live duplicate-agent bug reachable from a shipped gesture, and then something worse: respawn is reachable down at least five paths, and the proof gate I shipped sits on only one of them — not the primary one. Detaching a pane into a new tab makes a healthy remote shell report as dead, and the client starts a second one with agent resume while the original keeps running.

+
+ +

1 · How the counsel ran

+
+
Process and convergence⤢ zoom
+

Round 1

Grok · 10 findings

Opus · 8 findings

GPT seat · empty

shared brief
verified facts vs unverified leads

synthesis
both models independently say:
one record, not two

Round 2 · Grok 6 · Opus 7
→ key by leafId, not paneKey

Round 3 · Grok 5 · Opus 5
→ delete attach identity entirely

clean-check + fact-check

close-out
C1–C5

Round 1

Grok · 10 findings

Opus · 8 findings

GPT seat · empty

shared brief
verified facts vs unverified leads

synthesis
both models independently say:
one record, not two

Round 2 · Grok 6 · Opus 7
→ key by leafId, not paneKey

Round 3 · Grok 5 · Opus 5
→ delete attach identity entirely

clean-check + fact-check

close-out
C1–C5

+
+

The GPT seat returned nothing in all three rounds and was dropped. Grok ran through its CLI; the wrapper was instructed to report a failure rather than author findings, so its participation is real rather than assumed. Findings fell 10 → 6 → 5 and 8 → 7 → 5 across rounds — converging, not exhausted.

+ +

2 · The live bug

+

Four independently reasonable decisions compose into a shell-killer.

+
+
Detach a pane → duplicate agent⤢ zoom
+

user detaches a pane
into a new tab
shipped gesture

tabId changes;
relay still holds the old one
frozen at spawn

reattach → relay compares
and throws
PTY "x" not found (identity mismatch)

isSshPtyNotFoundError
regex /PTY ".+" not found/
matches

re-emitted as
SSH_SESSION_EXPIRED

isProvenSshSessionGoneError
returns true

clear binding →
startFreshColdRestoreAgentResume

second shell + SECOND AGENT RESUME
original shell still running

user detaches a pane
into a new tab
shipped gesture

tabId changes;
relay still holds the old one
frozen at spawn

reattach → relay compares
and throws
PTY "x" not found (identity mismatch)

isSshPtyNotFoundError
regex /PTY ".+" not found/
matches

re-emitted as
SSH_SESSION_EXPIRED

isProvenSshSessionGoneError
returns true

clear binding →
startFreshColdRestoreAgentResume

second shell + SECOND AGENT RESUME
original shell still running

+
+

The error string says the PTY was not found. It was found — that is how its identity got compared. The word “not found” is doing double duty for “absent” and “present but mismatched”, and a regex cannot tell them apart.

+ 
// my classifier — the last link
+export function isProvenSshSessionGoneError(error: unknown): boolean {
+  if (message.includes(SSH_SOURCE_RESTORE_REQUIRED_ERROR)) return false
+  return message.includes(SSH_SESSION_EXPIRED_ERROR) || /PTY ".+" not found/i.test(message)
+}
+// two functions away, unused by me:
+export function isSshPtyIdentityMismatchError(error: unknown): boolean
+
+

Why I missed it

+

I wrote this file to enforce “unknown is not dead,” and its doc-comment says “True only when the failure proves the session no longer exists.” I treated a message shape as proof. The discriminator I needed already existed in the same file. A test asserting “not-found ⇒ gone” passes happily, because the wrongness is in what the string means, not in what the code does with it.

+
+ +

The gate is on the wrong path

+

I shipped “respawn requires positive proof” and pinned it with a test. Grok traced every respawn path and found the proof check is bypassed on the main one, because the error never reaches a classifier — it is converted into a boolean first:

+ 
// pty-transport.ts — catches the error and RESOLVES
+return { id: options.sessionId, sessionExpired: true }
+
+// pty-connection.ts — respawns on the flag alone, no proof check
+if (connectResult?.sessionExpired) { /* clear binding */
+  startFreshColdRestoreAgentResume(...) }
+
+
Respawn paths vs the proof gate⤢ zoom
+

resolved as a flag

bare token

empty result

main process

thrown

reattach fails

how does the
failure travel?

sessionExpired: true
→ respawn
no proof check

isSshSessionExpiredError
→ respawn

no ptyId returned
→ respawn

lease marked expired
→ recoverTerminalPane

isProvenSshSessionGoneError
the gate I shipped

resolved as a flag

bare token

empty result

main process

thrown

reattach fails

how does the
failure travel?

sessionExpired: true
→ respawn
no proof check

isSshSessionExpiredError
→ respawn

no ptyId returned
→ respawn

lease marked expired
→ recoverTerminalPane

isProvenSshSessionGoneError
the gate I shipped

+
+
+

What this means for the shipped work

+

Two of my six changes are now known to be narrower than claimed. The keystroke fence is inert on the reattach path, and the respawn gate guards a minority of respawn routes. Both passed their tests because each test asserted the guard behaves correctly when reached — neither could observe that the guarded route is not the route production takes.

+
+ +

3 · What the counsel changed about my design

+ + + + + + + + +
My proposalCounsel’s designWhy theirs is better
Two records: ownership + orphan inventoryOne record. Orphans become a connect-time projection with no durable rowsA durable orphan row goes stale across exactly the disconnect during which the truth changed — and an orphan is only actionable while connected. Also deletes the two-record transaction my split needed.
Key by paneKey = tabId:leafIdKey by leafId; tab derived from the live layout, never storedA stored tabId is either frozen (names a tab the leaf left) or updated (desyncs from the relay’s copy). Both are live failures. Keying by the leaf deletes the transfer problem instead of managing it.
Keep attach-time pane identityDelete it — compare returned incarnation insteadThe apparatus is over-strict and under-strict at once: it rejects a live renamed pane, and accepts a different shell after a relay restart recycles the id. It fails at its own stated job.
“Unknown is not dead” as a principleSame rule, made structural: fate must be reported by the host, never inferred from timing, absence, or error-string shapeEvery defect found across three rounds is one violation of that clause. Mine was a slogan; theirs is a rule with an enforcement point.
+

Both models proposed the single-record collapse independently, which is the strongest signal available in a counsel and why it outweighed my version.

+ +

4 · The rule everything follows from

+
+
Ownership vs fate⤢ zoom
+

FATE — is the shell alive

non-durable

host-local

must be REPORTED by the host
never INFERRED by the client

OWNERSHIP — which pane owns which shell

durable

client-local

read synchronously
from disk while offline

not from timing
not from an absent entry
not from an error string

FATE — is the shell alive

non-durable

host-local

must be REPORTED by the host
never INFERRED by the client

OWNERSHIP — which pane owns which shell

durable

client-local

read synchronously
from disk while offline

not from timing
not from an absent entry
not from an error string

+
+

This is what makes the offline case work without an authority layer. Ownership is answerable with the network down, because it never depended on the network. Fate is never answered by the client at all.

+ +

5 · The record

+ 
/** Main-owned. TOP-LEVEL in PersistedState — never inside WorkspaceSessionState. */
+type PaneShellOwnership = {
+  leafId: string          // PRIMARY KEY — layout leaf UUID
+  hostId: ExecutionHostId // FIELD: 'local' | 'ssh:<target>' | 'runtime:<id>'
+  worktreeId: string
+  ptyId: string           // relay-native form for SSH — one namespace, always
+  incarnationId: string   // REQUIRED
+  relayInstanceId?: string
+  state: 'attached' | 'detached' | 'terminated'
+  createdAt / updatedAt / lastAttachedAt? / lastDetachedAt?
+}
+// Absent on purpose: no tabId, no paneKey, no hostEpoch, no second record.
+

Uniqueness is enforced on (hostId, ptyId, incarnationId), not ptyId alone — because relay ids are minted pty-1, pty-2… per relay process, so pty-1 recurs on every host after every relay restart. Round 1 had rejected a collision here as impossible; round 3 showed it is guaranteed, and the rejection was withdrawn.

+ +

6 · Orphans are a projection, not an authority

+
+
Why no durable orphan rows⤢ zoom
+

rejected

on connect

ask the host what it has

subtract what our records claim
incarnation-scoped

remainder = orphans
computed, never stored

feeds the existing cleanup affordance
never auto-killed

a durable orphan row would
go stale across the very disconnect
during which the truth changed

rejected

on connect

ask the host what it has

subtract what our records claim
incarnation-scoped

remainder = orphans
computed, never stored

feeds the existing cleanup affordance
never auto-killed

a durable orphan row would
go stale across the very disconnect
during which the truth changed

+
+ +

7 · What gets deleted

+ + + + + + + + + +
DeletedWhere
Attach-identity apparatusrelay comparison + throw, the error constants, the expected-identity derivation on both reattach paths, the renderer branch that treats a mismatch as proof
Lease↔binding reconciliationsupersession, ranking, deferral, rollback, dual matchers — the ~209 lines I added, plus the ~374 that preceded them
The restorable forkthe two contradictory predicates where absence of a lease meant opposite things depending on a sibling leaf
The 30-second grant arithmeticreplaced by an explicit record state
Never built at allhostEpoch and its clock arithmetic; a five-row evidence matrix; a resumed flag — all proposed in earlier rounds and killed before implementation
+
+

Net lines, honestly

+

Removed ≈900–1,250. Added ≈450–700. Net ≈ −300 to −700 — but only after a one-release legacy projection is dropped; during that release it is ≈ −100 to −450. Tests go up ≈900–1,500 for 25 oracles, several needing SSH + detach + relay-restart harnesses that do not exist. The synthesizer stated plainly it could not narrow the range further without doing the call-site rewiring.

+
+ +

8 · Verification

+

Models assert plausible file contents confidently, so every claim about existing code was checked.

+
+
78
Confirmed
+
12
Wrong
+
1
Unverifiable
+
+

Eleven of the twelve errors were citation-level and changed no conclusion. One was a real premise defect: the claim that abandoning a wrongly-attached shell is free. It is not — attach commits activation, clears pending output and may replay before it returns the incarnation the client would use to detect the mistake, and there is no detach RPC to undo it. That correction is folded in as C4.

+

I independently verified the duplicate-agent chain end to end, since it implicates my own code.

+ +

9 · Where the models disagreed

+ + + + + + + + +
ConflictStatus
Do shells survive a hard relay death on Windows/WSL?Open — adjudicated toward “dead” on POSIX evidence; the Windows case is not established
Does the 30-second recovery grant execute in production at all?Open — two reviewers read its comparison as unreachable for a real SSH pane; the covering test seeds a shape production cannot produce
Should relayInstanceId exist?Open — one would delete it and let incarnation carry the rule; the other keeps it because a re-minted id proves only that the relay restarted
Fix or delete the attach identity?Resolved — deletion, and the synthesizer went further than either reviewer
+ +

10 · Readiness

+

A final pass re-checked whether the close-out actually closed its own blockers. Verdict: STILL NOT READY.

+ + + + + + +
BlockerClose-out
The spawn reattach path still wraps any not-found as expired, so client-side auto-respawn survivesNOT CLOSED — C1 hardens the classifier, but the primary path never reaches a classifier. Every auto-respawn consumer must require proof, not just the one that catches a throw.
Migrated records have no incarnation, but the design requires one and throws on mismatchCLOSED, conditionally — C2/C3 work only if an absent recorded incarnation means learn and stamp, never mismatch. That rule must be written into both the algorithm and the type.
+ +

11 · Decisions for you

+ + + + + + + + +
#Decision
1The live bug. It is independent of the redesign and fixable in isolation. Fix it now as its own change, or fold it into the track?
2Visible detached panes. The design replaces an invisible 30-second auto-recovery with an explicit state and two actions (reattach / start a new shell). That is a deliberate product change — a user-initiated relay reset would now show detached panes rather than silently recreating shells.
3Release policy. A leaf removed while its shell lives releases the claim — the shell becomes orphan-eligible and adoptable, and is never auto-killed. Confirm you want liveness preserved over tidiness.
4Sequencing. Two preconditions block everything: one partition per (target, pane), and one bind producer. The second is also what would make the shipped fence non-inert.
+ +
+ Companion to report.html (evidence for the shipped work) and finalized-design.html (the two-plane decision this supersedes at the control plane). + Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
×
scroll to zoom · drag to pan · Esc to close
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md new file mode 100644 index 00000000000..30d99ae116e --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-brief-for-review.md @@ -0,0 +1,186 @@ +# Design brief — terminal session ownership & transport + +Review artifact. Repo: Orca (Electron + renderer + relay). Everything below marked +**[V]** was verified by reading the code at the cited location. Everything marked +**[R]** is agent-reported and only spot-checked. Treat **[R]** as a lead, not a fact. + +--- + +## 1. The product problem + +Orca runs terminals on four "routes": + +| Route | Provider | Survives app restart | +| --------------------- | ----------------------------------------------- | -------------------------- | +| local in-process | `LocalPtyProvider` | no | +| local daemon | `DaemonPtyAdapter` (swapped into the same slot) | yes | +| direct SSH | `SshPtyProvider` → remote relay | yes, within a grace window | +| paired remote runtime | not an `IPtyProvider`; runtime RPC | yes | + +Reported failures: + +1. **Pane cardinality growth (STA-3077).** SSH reconnect: 2 terminals → 19 → 20. + Most are panes the user never opened. Remote host fills with unused shells. +2. **Duplicate agent resume.** A coding agent gets resumed twice, two processes + writing one conversation file. One report reached five. +3. **Blast radius.** One PTY failing to re-prove its output stream could drop the + whole host connection, killing every pane, file transfer and git command on it. + +## 2. Verified facts (the evidence base) + +**[V] Dispatch is a substitution for daemon, a parallel registry for SSH.** +`src/main/ipc/pty.ts:932-941`: + +```ts +function getProvider(connectionId) { + if (!connectionId) return localProvider // local AND daemon — one slot + const provider = sshProviders.get(connectionId) // SSH — separate map +} +``` + +The daemon is installed by `setLocalPtyProvider(routedAdapter)` +(`src/main/daemon/daemon-init.ts:1031`) and adds **zero** branches to the shared +spine. The paired runtime is encapsulated behind a `PtyTransport`. SSH is neither — +it rides the local path carrying a `connectionId` tag. + +**[R] That tag leaks:** ~199 remoteness branches in main (52 in `pty.ts`, 143 in +`orca-runtime.ts`), ~191 in the renderer, plus 25 `instanceof` provider checks. +The `remote:` predicate is re-declared 5 times in the renderer. + +**[V] The transport dropped the half that makes it persistent.** +`src/main/ssh/relay-protocol.ts:34`: + +```ts +export const MessageType = { Regular: 1, KeepAlive: 9 } as const +``` + +The 13-byte frame header was copied from a well-known persistent protocol, but +only 2 of its 9 message types were kept. Dropped: `Control`, `Ack`, `Disconnect`, +`ReplayRequest`, `Pause`, `Resume`, `None`. The surviving ACK field feeds only +`unackedTimestamps` (`ssh-channel-multiplexer.ts:107,448,596`), a seq→timestamp map +read solely by the death timer. **Nothing is retained at the transport layer; +nothing is replayed.** Consequence: three layers above it each rebuilt +retain-and-replay independently (SSH credit ledger, runtime-RPC source-range +ledger, daemon batcher — the last deliberately lossy). + +**[V] Method layer is transport-agnostic.** `runtime/rpc/methods/terminal.ts` and +`git.ts` have 0 SSH references; `files.ts` has 27. + +**[V] Three durable records exist, plus a fourth copy on the host.** + +| | key | pane identity is | +| ------------------------------------- | ------------------------- | ----------------------------- | +| PTY binding (`WorkspaceSessionState`) | `(hostId, tabId, leafId)` | **the key** | +| `SshRemotePtyLease` | `(targetId, relayPtyId)` | **optional payload** | +| `SshPtyConsumerRecovery` | `targetId` | n/a (per-target resume token) | +| relay `attachIdentity` | per PTY, in-memory | compare-only | + +**[R] ~374 lines of pure lease↔binding reconciliation in `persistence.ts`**, and 9 +enumerated states where the records can disagree. Example of an independent latent +bug: absence of a lease means "restorable" in `isRestorablePtyBinding` but "not +restorable" in `hasRestorableSshRemotePtyLease` — so whether a binding survives +depends on whether a _sibling leaf_ had a binding. + +**[V] The relay holds nothing durably.** Its PTY table is +`private ptys = new Map()` (`src/relay/pty-handler.ts:356`). +No relay-side persistence of PTY state exists. `pty.serialize`/`revive` hand state +to the _client_ and take it back; `revive` requires the original PIDs to still be +alive. On PTY exit the entry is deleted with no tombstone. + +**[V] Pane identity is write-only over the wire.** `listProcesses` and `attach` +never return `paneKey`/`tabId`/`leafId`; `attach` only compares and throws +`identity mismatch`. The one RPC that returns identity (`pty.serialize`) has +**zero production callers** — test files only. + +**[V] The client's lease is consulted from a synchronous, offline, disk-only path.** +`hasRestorableSshRemotePtyLease` is called inside `setLocalWorkspaceSession` +(`src/main/persistence.ts:6465`), a synchronous `void` method invoked on every +renderer session publish including during quit (`:6287`, `:6351`). + +## 3. Alternatives already refuted + +**A. Merge the lease into the pane binding.** Blocked by two load-bearing +properties: a lease must be able to name a live shell **no pane owns** (the actual +STA-3077 field state; reattach enumerates these), and a lease **outlives** the +binding — expiry deletes the binding and then authorizes recreating a shell for 30s +(`SSH_PANE_RECOVERY_GRACE_MS`, `orca-runtime.ts:1771`). + +**B. Delete the lease; make the remote host authoritative.** Refuted. The host has +no durable state, cannot be queried for pane identity, has no notion of "recently +died, replacement authorized", and is by construction unreachable exactly when the +record is needed. Decisive: you cannot replace a synchronous local disk read during +quit with an RPC to a host that is down. + +**C. A second "authority architecture" beside the existing one** (+60,903 lines, +394 new files). Rejected earlier: it fixed none of the three root causes. + +## 4. The design under review + +### Plane 1 — data plane (adopted from a separate proposal) + +Restore a delivery guarantee at the transport, then collapse what was rebuilt above +it. Roughly: capability handshake; one delivery-identity type; one credit-ledger +implementation parameterised by unit and limit; one chunk-and-ack primitive; one +lane vocabulary; binary PTY payloads reusing the binary terminal frame Orca already +ships to mobile; prebuilt native binaries so first connect stops needing a compiler; +then teach the relay to listen on a loopback WebSocket, forward that port, and let +the _same_ client serve the SSH route — after which SSH is "launch + access" only. + +### Plane 2 — control plane (proposed here; the reviewed contribution) + +The lease answers two different questions with one record keyed for neither: + +- "which shell does this pane own?" → wants a **pane** key +- "which shells exist that nobody owns?" → wants a **pty** key + +Proposal: split by question. + +- **Ownership record**, keyed `(host, pane)` — the _same_ key local already uses. + Transport-specific payload (lease state + timestamps for SSH; nothing extra for + local). Makes "one pane, one live shell" structural rather than enforced. +- **Orphan inventory**, keyed `(host, ptyId)` — live shells with no owner. Feeds the + reattach work list and the disconnected-cleanup UI. +- The 30s recovery grant becomes a terminal state on the pane-keyed record. +- Offline consumers keep reading local disk synchronously, unchanged. + +Claimed payoff: the ~209 lines of supersession/ranking/rollback/dual-matchers exist +_only_ to bolt a pane-keyed constraint onto a pty-keyed record, and become +unnecessary. + +## 5. Historical defect, now resolved + +This defect was resolved by the shared `bindPaneShell` producer used by relay reattach and both +spawn handlers. Its mutation proof is recorded in +[S5](./new-design-goalposts.md#s5--the-superseded-pane-fence-is-live-on-the-reattach-path--proven). + +## 6. Hard constraints + +- **Cross-platform**: macOS, Linux, Windows (+ WSL). No `echo $$`/`ps` assumptions. +- **Folder workspaces**, not only git worktrees. +- **Remote wire compatibility**: clients and remote hosts update independently; + mixed versions are normal. New optional field = safe; new stream opcode must be + capability-negotiated (decoders drop unknown opcodes silently); changing what a + host publishes reaches old clients with no wire change. +- **Git 2.25 baseline** for git-touching work. +- Durable on-disk state is also read by pinned older builds in cross-version tests. +- "Unknown is not dead": disconnect, timeout, absent inventory, and retry + exhaustion must never authorize destroying a process. +- No `max-lines` lint disables. + +## 7. What the review must answer + +1. **Correctness holes.** Which cases does the Plane 2 design get wrong or fail to + cover? Specifically: multi-window / multi-device on one host; app restart while + disconnected; relay restart; host reboot; PTY exit during disconnect; two panes + racing for one shell; migration from existing on-disk state; a pane deleted while + its shell lives; the same worktree open in two windows. +2. **Simplicity.** Is there a _simpler_ design with the same correctness? Simpler + means fewer code paths and fewer concepts, not merely fewer lines. Is the + two-record split actually necessary, or is there a single-record formulation that + survives the section-3 refutations? +3. **Migration.** Re-keying a durable record needs a story for existing state, + partial writes, and rollback. +4. **Testability.** What oracle would fail if each guarantee were removed? A design + whose guarantees cannot be falsified by mutation is not finished. +5. **Is Plane 2 even the right cut?** Attack the premise that data plane and control + plane are separable here. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html new file mode 100644 index 00000000000..8718846bd98 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html @@ -0,0 +1,235 @@ + + + + + + Terminals that don't duplicate — the plain-English design + + + +
+
+

Why terminals duplicate,
and how we stop it

+

Plain English, no code. Two bugs, two fixes, then one decision for you.

+
+ +
+ The whole design in one line:
+ Keep our own notes about which terminal is whose — and never guess whether a program died. +
+ +
Setup
+

What's behind one terminal

+

When you open a terminal on a remote machine, there are three things, not one.

+
+
The three pieces⤢ zoom
+

THE PANE
the box on your screen

THE NOTE
this pane goes
with that program

THE PROGRAM
really running on
the other machine

THE PANE
the box on your screen

THE NOTE
this pane goes
with that program

THE PROGRAM
really running on
the other machine

+
+

Your network blips. The pane stays on screen. The program keeps running over there. Only the connection between them breaks.

+

So reconnecting is one job: match the notes back up to the programs. Both bugs below are that matching going wrong.

+ +
Bug one
+

The notes piled up

+

A real user report: 2 terminals became 19, then 20. Most were panes they never opened.

+
+
Why they multiplied⤢ zoom
+

you have 2 terminals

blip → reconnect

Orca writes a new note
and keeps the old one

another blip

another note.
old ones still kept.

reconnect restores EVERY note.
one pane per note.
19 panes

you have 2 terminals

blip → reconnect

Orca writes a new note
and keeps the old one

another blip

another note.
old ones still kept.

reconnect restores EVERY note.
one pane per note.
19 panes

+
+

The cause is boring and completely fixable: the notes were filed under the program's name, not the pane's. One pane could collect unlimited notes, and nothing threw the old ones away.

+ +

The fix: one slot per pane

+
+
Filed by pane instead⤢ zoom
+

after · filed by pane

pane A

its one note

before · filed by program

pane A

note 1

note 2

note 3

after · filed by pane

pane A

its one note

before · filed by program

pane A

note 1

note 2

note 3

+
+

Piling up is no longer something we prevent — there is one slot, so it cannot happen. That's the difference between a rule you enforce and a shape where the bug has nowhere to live.

+ +
+ One detail that matters +

The note is filed under a permanent serial number stamped on the pane when it's created — not under "which tab it's in". Tabs change: you can drag a pane into another tab any time. A serial number never does. File things under what doesn't move.

+
+ +
Bug two
+

"I can't find it" was read as "it died"

+

This is the worse one, because it's how you end up with two AI agents writing into the same conversation.

+
+
Drag a pane to a new tab, then reconnect⤢ zoom
+

you drag a pane
into a new tab

the other machine still has
the OLD tab written down

on reconnect it compares, sees
a difference, and replies:
no such terminal

Orca reads that as
the program died

starts a replacement,
and resumes your agent again

TWO agents writing to one
conversation. the first one
never stopped.

you drag a pane
into a new tab

the other machine still has
the OLD tab written down

on reconnect it compares, sees
a difference, and replies:
no such terminal

Orca reads that as
the program died

starts a replacement,
and resumes your agent again

TWO agents writing to one
conversation. the first one
never stopped.

+
+ +
+ The trick in it +

The other machine did find your program. Comparing is how it noticed the tab was different. But it answered "not found" — so a message meaning "it's alive and I'm being fussy" looked exactly like "it's gone."

+
+ +

Fix, part one: stop comparing tabs

+

Moving a pane between tabs is a normal thing you're allowed to do, so it must never make a terminal unreachable. That comparison is removed.

+

It was also failing at its real job anyway. It existed to catch a different problem — the remote machine reusing an old ID for a brand-new program, like an apartment number being handed to a new tenant. It never caught that. So we check the tenant instead of the apartment number: catches the real case, stops rejecting panes that merely moved.

+ +

Fix, part two: never guess about death

+

A replacement terminal starts only on actual proof the old program is gone.

+
+
What counts as proof⤢ zoom
+

did the program die?

we watched it exit

the same helper that started it
says it's gone

can't reach the machine

it timed out

missing from a list

confusing error message

PROOF
start a replacement

NOT PROOF
leave it running.
ask the user.

did the program die?

we watched it exit

the same helper that started it
says it's gone

can't reach the machine

it timed out

missing from a list

confusing error message

PROOF
start a replacement

NOT PROOF
leave it running.
ask the user.

+
+

The second proof is the clever one. The helper on the remote machine keeps its list of running programs only in memory. So if that same helper — same process, not a restarted one — says a program isn't in its list, it must have exited. There is nowhere else it could be.

+

And if the helper restarted, it remembers nothing, so we say we don't know. We never treat a fresh helper's empty list as news that your programs died.

+ +
Result
+

What you'd actually notice

+ + + + + + + + +
SituationTodayAfter
Network drops and comes backTerminals multiply; the remote machine fills with junkYour terminals come back. Nothing invented.
You drag a pane to a new tabA second agent resumes onto the same conversationJust works.
Orca genuinely can't tell if a program survivedSilently starts a new onePane says "disconnected", with two buttons
Leftover programs on a remote machineInvisible, pile up foreverListed with folder and name — you choose what to close
+ +
+
The one visible change⤢ zoom
+

Orca isn't sure whether
your program survived

TODAY
quietly starts a new one.
you may not notice until
there are two agents.

AFTER
pane shows 'disconnected'
[ try again ] [ new terminal ]

Orca isn't sure whether
your program survived

TODAY
quietly starts a new one.
you may not notice until
there are two agents.

AFTER
pane shows 'disconnected'
[ try again ] [ new terminal ]

+
+ +
+ This is the decision I need from you +

Today, uncertainty gets resolved silently and wrongly. After this, uncertainty becomes visible — a disconnected pane with two buttons.

+

Slightly more friction in a rare case, in exchange for never resuming your agent twice. I think it's clearly right, but it changes what people see, so it's your call.

+
+ +
Plan
+

The order we'd build it

+

Six steps. Each ships on its own and can be undone on its own.

+ + + + + + + + + + +
StepWhat it does
1Stop the tab comparison killing live terminalsFixes the double-agent bug. Small, self-contained, helps immediately.
2Stop faking "the program exited"Today a failed reconnect tells the pane the program exited — a claim we can't back up. Ships with the two-button UI.
3Write the notes in one placeThey go to two different places today, which makes the cleanup silently do nothing. This is the actual cause of 2 → 19.
4One piece of code writes the notesTwo paths write them and a third forgets to — which is why an existing safety check never runs during reconnect.
5Check whether an old recovery path ever runsTwo reviewers believe it's unreachable. Measure before building on it.
6The proof ruleOnly if step 5 shows it's needed.
+ +
+

The five words I couldn't avoid

+ + + + + + + + +
PaneOne terminal box on your screen.
ProgramThe shell or agent actually running — often on another machine.
NoteOrca's record that a pane goes with a program. Saved to disk, so it survives a restart.
HelperThe small program Orca installs on a remote machine to run terminals there.
OrphanA program still running that no pane claims. We list them; we never close them for you.
+
+ +
+ Why this is smaller, not bigger +

It removes roughly 900–1,250 lines and adds roughly 450–700. Most of what goes away exists only to patch up disagreements between notes filed different ways. Once there's one note per pane, that code has nothing left to do.

+
+ +
+ The detailed versions: counsel-design.html — how this was reviewed and what it cost. + report.html — evidence for the work already written. + Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
×
scroll to zoom · drag to pan · Esc to close
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md new file mode 100644 index 00000000000..d6b474c93a8 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md @@ -0,0 +1,256 @@ +FINAL DESIGN — one leaf-keyed ownership record; pane identity deleted from the attach path; death proved only by the relay process that spawned the shell + +===================================================================== +D0. SCOPE AND THE ONE RULE EVERYTHING FOLLOWS FROM +===================================================================== +Problem (from the brief): SSH reconnect grows pane cardinality (2 -> 19 -> 20, STA-3077); a coding agent gets resumed twice into one transcript (RC2); one PTY's failure can drop a whole host connection (RC3, out of scope — Plane 1). + +One rule, applied without exception: + +Ownership ("which shell does this pane own?") is DURABLE, CLIENT-LOCAL, and read +synchronously from disk while offline. Evidence about a shell's FATE is +NON-DURABLE, HOST-LOCAL, and must be REPORTED by the host, never INFERRED by the +client from timing, from the absence of an entry, or from the shape of an error +string. + +Every defect found across three review rounds is one violation of the second clause. The design's job is to remove all of them, not to add an authority layer. + +SCOPE DECLARATION (grok, low): the record is host-agnostic (`hostId` is a field). The _death rule_'s row 2 is relay-specific. Local and daemon panes reach `terminated` only through row 1 (an exit observed in-process, which is how they already work) and never through row 2. That is stated rather than left implicit. + +===================================================================== +D1. THE DURABLE RECORD +===================================================================== + +```ts +/** Main-owned. TOP-LEVEL in PersistedState — never inside WorkspaceSessionState. */ +type PaneShellOwnership = { + leafId: string // PRIMARY KEY. Terminal-layout leaf UUID. + hostId: ExecutionHostId // FIELD: 'local' | 'ssh:' | 'runtime:' + worktreeId: string // FIELD. Opaque FULL string incl. any `::workspace:` suffix. + ptyId: string // RELAY-NATIVE form for SSH (toRelaySshPtyId). One namespace, always. + incarnationId: string // REQUIRED. Relay mints one per pty (relay/pty-handler.ts:1525). + relayInstanceId?: string // The relay PROCESS that owned it, as reported at bind time. + state: 'attached' | 'detached' | 'terminated' + createdAt: number + updatedAt: number + lastAttachedAt?: number + lastDetachedAt?: number +} +// Storage: PersistedState.paneShellOwnershipByLeafId: Record +``` + +Note what is NOT here: no `tabId`, no `paneKey`, no `hostEpoch`, no `resumed`, no second (pty-keyed) record. + +WHY `leafId` AND NOT `paneKey`. `makePaneKey(tabId, leafId)` returns `${tabId}:${leafId}` and the repo states at src/shared/stable-pane-id.ts:47-49 that "only the leaf UUID is remint-stable pane identity (the tab half changes on pane break-out)" — which is why `isEquivalentPaneKey` exists at :50-57. The mutation is a shipping gesture: `detachTerminalPaneToTab` (src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.ts:245-310) moves a LIVE pane and its ptyId into a new tab via `createTab(..., initialPtyId)` while explicitly not killing the PTY (:275-276). `leafId` is a UUID and is never remapped once it is one (src/main/persistence.ts:1858-1872 skips ids already passing `isTerminalLeafId`). Keying by the leaf deletes the transfer problem instead of managing it. + +WHY NO `tabId` AT ALL (this is the round-3 change). A stored tabId is either frozen (and then it names a tab the leaf no longer lives in) or updated (and then it desynchronizes from the relay's frozen copy). Both are live production failures — see D2. The tab is LOCATION, derived from the live layout at the moment it is needed; it is never stored and never trusted. + +WHY TOP-LEVEL. `session:set` / `session:set-sync` (src/main/ipc/session.ts:13, :31) accept a whole `WorkspaceSessionState` from a renderer with no revision or fencing token; the clobber guards at src/main/persistence.ts:6379, :6384-6476 exist only because of that. `sshRemotePtyLeases` is already top-level. The KEY is shared with the layout; the STORAGE LOCATION is not. + +UNIQUENESS. At most one non-terminal record may name a given `(hostId, ptyId, incarnationId)`, enforced on write through a reverse index. Not ptyId alone: leases store the relay-native id (`getRelayPtyIdForSshLeaseStorage` -> `toRelaySshPtyId` strips the connection scope, src/main/persistence.ts:6641-6643, src/shared/ssh-pty-id.ts:58-67) and relay ids are `pty-${this.nextId++}` with `nextId = 1` per relay PROCESS (src/relay/pty-handler.ts:356-357, :1440-1443), so `pty-1` recurs on every host after every relay restart. Round 1's rejection ("a cross-target collision on ptyId is impossible") is withdrawn; it is guaranteed, not impossible. + +===================================================================== +D2. DELETE PANE IDENTITY FROM THE ATTACH PATH (the largest round-3 change) +===================================================================== +Both reviewers independently attacked the attach-time identity comparison. Verified, and it is worse than either stated: the apparatus is simultaneously over-strict and under-strict, and it is a live duplicate-agent bug today. + +VERIFIED FACTS + +- The relay compares frozen strings: `attachIdentityMismatches` (src/relay/pty-handler.ts:334-339) does full-string `expected.paneKey !== managed.paneKey || expected.tabId !== managed.tabId`; `attachIdentity` is captured once at spawn from `ORCA_PANE_KEY`/`ORCA_TAB_ID` (:1512-1517, :1446). On mismatch it throws `PTY "" not found (identity mismatch)` (:1616-1626). +- `isSshPtyNotFoundError` is a message regex `/PTY ".+" not found/i` (src/main/providers/ssh-pty-errors.ts:12-15), so it is TRUE for the mismatch string. `reattachSshPtySession` re-emits it as `SSH_SESSION_EXPIRED: SSH_PTY_IDENTITY_MISMATCH` (src/main/providers/ssh-pty-session-reattach.ts:218-226). +- The renderer's `isProvenSshSessionGoneError` returns TRUE for anything containing `SSH_SESSION_EXPIRED` and for the raw not-found regex, and IGNORES the mismatch marker (src/renderer/src/components/terminal-pane/reattach-failure-classification.ts:24-30). Its two call sites then clear the binding and call `startFreshColdRestoreAgentResume` (pty-connection.ts:8839-8847, :9086-9097) — a second shell and a SECOND AGENT RESUME while the original shell keeps running. That is RC2, on a live process, reachable from a shipped gesture. +- The spawn-path reattach builds the expected identity from the CURRENT options (`args.options.paneKey ?? env.ORCA_PANE_KEY`, ssh-pty-session-reattach.ts:66-68), i.e. the current tab — so after a detach it mismatches. The connect-path reattach builds it from the lease's stored tabId (`expectedIdentityForLease`, src/main/ssh/ssh-relay-session.ts:190-210, consumed at :2201-2210), which is never re-keyed on detach — so it matches, attach succeeds, and then `persistPtyBinding(tabId = old tab, mayCreate:false)` refuses because the leaf is no longer in that tab's layout (src/main/persistence.ts:6821-6877 sets `terminalMembershipChanged` and returns false), leaving the shell running unbound (ssh-relay-session.ts:2513-2521). Grok's "scissors" is confirmed on both blades. +- The apparatus does not even do its own stated job ("generation resets can reuse PTY IDs; reject conflicting identities", pty-handler.ts:1615): after a relay restart that recycles `pty-3` for the SAME pane, paneKey and tabId both match and a different shell is accepted. + +RESOLUTION — delete the comparison; do not replace it with another comparison. + +1. Relay: delete `attachIdentity`, `attachIdentityMismatches`, the `params.expectedPaneKey`/`expectedTabId` handling and the `(identity mismatch)` throw. `ManagedPty.paneKey`/`tabId` STAY — they are documented as separate from attach identity (pty-handler.ts:139-143) and are used by the exit listener (:765) and, newly, by listProcesses reporting (D4). paneKey stops being an authorization INPUT and becomes a reported HINT. +2. Main: delete `ExpectedPtyIdentity`, `expectedIdentityForLease`, `expectedIdentityByPtyId`, the `expectedIdentity` parameter threaded through `attachPtyWithRetry`, `isSshPtyIdentityMismatchError`, `SSH_PTY_IDENTITY_MISMATCH_ERROR`, the marker append, and the mismatch branch at ssh-relay-session.ts:2628-2634. +3. Renderer: `isProvenSshSessionGoneError` keeps only proofs. (After step 1/2 the mismatch string ceases to exist; the renderer fix ships FIRST and independently, because it must protect users running against relays that have not been redeployed.) +4. Recycled-id rejection moves to two places that already hold exact data: + (a) PROCESS GATE, once per connection: if the owner grant reports a `relayInstanceId` different from the one recorded on a record, that record's ptyId is not attachable at all. One comparison covers every pty on the host. Within a single relay process ids never recycle (`nextId` is monotonic), so this is complete against new relays. + (b) PER-ATTACH GUARD, zero wire change: EVERY `pty.attach` success path already returns `incarnationId` (src/relay/pty-handler.ts:1646, :1653, :1668, :1676), and the client already parses and remembers it (`requestSshPtyAttach` -> `rememberPtyIncarnation`, ssh-pty-session-reattach.ts:74-100). Compare it to `record.incarnationId`; on mismatch, throw — the existing `activationLease.rollback()` already runs on a thrown error — and classify `detached`, never death. `RecentPtyOutputBuffer.read()` is explicitly non-mutating (src/main/runtime/recent-pty-output-buffer.ts:75-85), so abandoning a wrongly-attached shell does not consume its replay. + This is strictly stronger than the frozen-string check (a per-pty UUID vs a pane label), needs NO new request field, and works against OLD relays — which a new `expectedIncarnationId` request parameter would not. +5. The reattach BIND resolves the tab from the live layout: find the tabId whose `terminalLayoutsByTabId[*].root` contains this `leafId`. If no layout contains it, refuse exactly as today (leave the shell running unbound, "no durable pane owns this"). `mayCreate:false`'s guarantee is preserved; its dependence on a stored tabId is deleted. + +ONE IDENTITY RULE, EVERYWHERE: the leaf is identity; the tab is derived location; the incarnation is shell identity; the relay instance is process identity. There is no fourth. + +===================================================================== +D3. THE DEATH RULE — two rows, one new field +===================================================================== +`resumed` and `hostEpoch` are deleted and must not be built. + +- `resumed: replaces !== null` (src/shared/pty-consumer-session.ts:275); `expireOwner()` nulls the incumbent after `PTY_CONSUMER_OWNER_GRACE_MS = 30_000` (:281-287, contract :2), after which admission takes `if (!current) return this.newOwner(...)` (:215-216) and `resumed` is false regardless of proof. Unreachable for quit, sleep, overnight, reboot — the normal case. Its own contract comment says it means "the client's checkpoints for the previous claim no longer apply" (pty-consumer-session-contract.ts:49-52): a delivery-checkpoint flag, not liveness. +- `hostEpoch = Date.now() - os.uptime()*1000` under exact equality is a reconstructed estimate; second-granularity uptime, NTP steps and suspend/resume make it differ on an unrebooted host, and "differs" was the grant-YES row — it fails OPEN toward respawn. Repo precedent settles it: `readCurrentDaemonReadyIdentity` reads `/proc/sys/kernel/random/boot_id` on Linux only and returns bare `{ startedAtMs }` elsewhere (src/main/daemon/daemon-ready-identity.ts:11-13). + +THE ONE NEW GRANT FIELD: `relayInstanceId?: string` on `PtyConsumerSessionGrant` (src/shared/pty-consumer-session-contract.ts:43-59) — a `randomUUID()` minted once at relay process start (`randomUUID` is already imported at relay/pty-handler.ts:6). It is genuinely new information: the grant's existing `serverBuildId` is `launchVersion` (src/relay/relay.ts:694-696), a build string that is identical across restarts of the same build. + +| Condition | Transition | Respawn grant | +| ---------------------------------------------------------------------------------- | ------------ | -------------------- | +| PTY exit observed on a live attached stream | `terminated` | yes | +| attach returns not-found AND the grant's `relayInstanceId` EQUALS the recorded one | `terminated` | yes (subject to E-1) | +| attach returns an incarnation different from the recorded one | `detached` | no | +| `relayInstanceId` absent on either side, or differing | `detached` | no | +| any other error (today's `restoreRequired` path, ssh-relay-session.ts:2612-2620) | `detached` | no | + +Row 2 is sound and UNBOUNDED IN TIME: `this.ptys` is the relay process's only store (pty-handler.ts:356) and entries are deleted on exit, so "the same process that minted this pty no longer has it" means it exited. It is sound on Windows too, because the relay OBSERVED the exit — unlike any rule that reasons from a relay restart. A restarted relay reports a different id and we correctly claim no knowledge. + +SHIP GATE (grok H4): row 2 grants a respawn on a not-found detected by a message regex. One shared normalization helper (`toRelaySshPtyId`) must be applied on every attach / mark / record / comparison path, and the record stores exactly one namespace (D1), so a namespace mismatch cannot manufacture a false not-found. Row-2 grant-yes does not ship until E-1 is green. + +STOP FABRICATING THE EXIT. `handlePtyReattachFailure` currently sends `pty:exit { code: -1 }` (ssh-relay-session.ts:2643-2647) plus `clearProviderPtyState` + `deletePtyOwnership` + `markSshRemotePtyLease('expired')` on a plain not-found. On every `detached` row this is a lie about a process the client knows nothing about. Remove the whole block and route the unknown case into the branch that ALREADY EXISTS three lines above it: `pending.restoreRequired = 'reattachAttemptsExhausted'` + `wakeRecovery` (:2612-2620), which is non-destructive by construction. This is a branch COLLAPSE, not a new signal — which is also the answer to the objection that removing only the send turns a wrong signal into no signal. + +===================================================================== +D4. ORPHANS — a connect-time projection, never an authority +===================================================================== + +``` +orphans(host) = successful listProcesses(host) + MINUS { (ptyId, incarnationId) named by a non-terminal record on this host } +``` + +Computed at connect/reattach, held in memory for the connection's life, discarded on disconnect. No durability, no migration, no rollback. + +WHAT `listProcesses` IS: it iterates only `this.ptys` (pty-handler.ts:1894-1911) — the current relay process's Map. No durability, no OS scan. It is therefore (i) the cleanup-UI source and (ii) a SUPPLEMENT to the reattach work list, never the sole discovery authority. The work list stays `getPtyIdsForConnection(target) UNION` the non-terminal records (today's `leasedPtyIds` shape, ssh-relay-session.ts:2211-2219). + +ABSENCE IS NEVER AUTHORITATIVE. The repo already adjudicated this with an issue number: `pty:listSessions` documents "Absence is authoritative only from a provider that serializes claims — otherwise it is 'unknown', never 'absent' (#8459)" (src/main/ipc/pty.ts:7343-7347), and `SshPtyProvider.hasPty` returns `null` before a completed listing "a miss there is ignorance about the host, not a dead PTY" (src/main/providers/ssh-pty-provider.ts:308-312). The only aggregating caller launders every remote failure with `provider.listProcesses().catch(() => [])` (ipc/pty.ts:7333), and `PtyProcessListAdmission.admit` throws on capacity and on `agent_session_ownership_unknown` (src/main/providers/pty-process-list-admission.ts:60-99). So: a rejected or truncated listing yields orphans = UNKNOWN (empty UI, no claims, no retirement), never orphans = everything. + +THE ONE NEW RESPONSE FIELD: `PtyProcessSummary` gains optional `paneKey`. The relay already holds it (`ManagedPty.paneKey`, pty-handler.ts:1446, :139-140) and omits it from the summary (:1899-1908). Additive optional response field — safe under docs/reference/remote-wire-compatibility.md, no opcode, no negotiation. It does NOT reach the client for free: `PtyProcessInfo` has no `paneKey` (src/main/providers/pty-process-info.ts:4-16) and `admit` REBUILDS an allowlist (pty-process-list-admission.ts:106-115). Three edits — relay summary, `PtyProcessInfo`, allowlist — and the oracle asserts survival THROUGH `admit`. `incarnationId` and `worktreeId` already traverse that whole chain, which is what makes the incarnation-scoped subtraction free. + +ADOPTION matches `parsePaneKey(reported.paneKey)?.leafId` — never the full string, because the relay's copy is frozen at spawn and names a tab the leaf may have left. Because the record is leaf-keyed this is a direct lookup, not fuzzy equivalence; `isEquivalentPaneKey` is not imported here. Adopt is a CAS: write `leafId -> (ptyId, incarnationId)` only if no non-terminal record exists for that leaf AND none on this host names that `(ptyId, incarnationId)`. Exactly one of two concurrent adopts wins; the loser gets a typed refusal. + +KILL POLICY (replaces the "owned elsewhere" classification, which is DELETED). Both reviewers attacked `worktreeId`-based classification from opposite directions and both are right, because `worktreeId = ::` and `repoId` is a client-local `randomUUID()` minted at repo add (src/shared/worktree-id.ts:15-28; src/main/runtime/orca-runtime.ts:18729). So "unknown worktreeId" means "not in this install's current state file": it wrongly EXEMPTS the user's own shells after a repo re-add, profile transfer or state reset, and it wrongly INCLUDES another device's shells whenever the two installs share state. One rule replaces it: the orphan list never auto-kills, never bulk-kills, and never kills as a side effect of any reattach or cleanup pass. Kill is an explicit per-shell user action, with cwd, title, worktreeId and reported paneKey shown. No classifier, no new field, and it is safe in both directions. + +OLD-RELAY DEGRADATION: a relay that omits `paneKey` yields unclassified orphans that are listed and never killed — exactly today's behavior. + +===================================================================== +D5. RECORD LIFETIME — three exits, none of them "absent from a list" +===================================================================== + +1. RETIRE (delete). The record reached `terminated` (row 1 or row 2 of D3, or explicit user close via `retireExitedPty`, ssh-relay-session.ts:2153-2172). Terminal records are garbage-collected after a bounded retention (long enough to serve any surviving grant window) so they cannot mask a recycled `pty-1`. There is NO retire-on-absence clause: the case it was written for (shell exited during a disconnect, relay still up) is already covered by row 2 — the attach itself proves the exit — so deleting the clause costs nothing and removes a direct contradiction with #8459. +2. RELEASE (keep the shell, drop the claim). Fired ONLY from the explicit pane-close and tab-delete code paths — never from "this publish did not contain the leaf". Detach is not atomic across layouts: `createTab` installs an empty layout, then `setTabLayout(sourceTabId, ...)` removes the leaf, then `setTabLayout(tab.id, ...)` adds it (terminal-pane-tab-detach.ts:298-300), so between the last two no layout contains the leafId, and any `session:set` flush in that window would look like a durable removal. Released records are cleared to a non-claiming state: the shell becomes orphan-eligible (visible, adoptable) and is deliberately NOT `terminated`, so it can never authorize a respawn. +3. EXPLICIT USER CLOSE -> `terminated`, and the record dies with the shell. + +RESIDUAL, STATED HONESTLY: a shell that died while the relay ALSO restarted leaves a `detached` record with no proof available. It stays restorable until the user acts. That is the conservative direction and it is bounded by the affordance below, not by a timer. + +PRODUCT AFFORDANCE, REQUIRED IN THE SAME PR. A `detached` pane renders as disconnected with two explicit actions — "reattach (retry)" and "start a new shell" (the latter retires the record). Nothing infers death, nothing auto-spawns. Per AGENTS.md this must follow docs/STYLEGUIDE.md. COPY CONSTRAINT: the UI must never assert that a shell is dead. The claim "after a relay crash the shells are almost certainly dead" is REMOVED from this design: it rests on POSIX master-fd close SIGHUPing the foreground group, and on Windows the relay is deliberately launched through WMI because "Windows sshd kills the exec channel's process tree on close" (src/main/ssh/ssh-relay-deploy.ts:1805), with ConPTY and no SIGHUP. No rule in this design concludes anything from a relay restart, so the design is correct either way; only the copy had to change. + +===================================================================== +D6. HISTORICAL — one partition per (target, pane). SUPERSEDED. +===================================================================== +This migration proposal was rejected. The shipped containment keeps the existing persistence +planes and gives the pane binding one local home; see [S4](./new-design-goalposts.md#s4--one-partition-per-target-pane--proven). + +SSH pane bindings live in two partitions today. Readers and writers that consult ONLY `ssh:`: `resolvePersistedStablePaneOwner` (src/main/ipc/pty.ts:676-677), `retirePersistedStablePaneOwner` (:760-761, :783), the CAS write `persistPtyBinding(..., expectedBinding)` (:833), and the two spawn upserts (:5108, :6493). Against that, the relay reattach write has no hostId (ssh-relay-session.ts:2500-2511 -> `resolveHostId(undefined)` -> `LOCAL_EXECUTION_HOST_ID`, persistence.ts:6243-6246) and the renderer keeps SSH worktrees in `local` deliberately (workspace-session-host-persistence.ts:167-173). `durablyBoundPtyIdForPane` hedges ssh-first-then-local (persistence.ts:7269-7277), so `supersedeSiblingLeasesForPane` early-returns when the bound pty differs (:7229-7232) — supersession silently no-ops and both leases stay live. That is the STA-3077 mechanism itself. + +FIX: one accessor `sessionForSshPaneBindings(connectionId): WorkspaceSessionState` returning the `local` partition, routed through by EVERY reader and EVERY writer above, flipped in a single atomic PR — not five write-site edits. `local` wins because the only publisher of pane membership writes there and `mayCreate:false` is evaluated there. A one-time fold moves `ssh:.terminalLayoutsByTabId[*].ptyIdsByLeafId` and `tabsByWorktree[*].ptyId` into `local`, preferring `local` on conflict; the loser is NOT demoted to orphan-candidate by ptyId alone (recycled ids collide) — it carries `incarnationId` or is dropped. + +===================================================================== +D7. HISTORICAL — one bind producer. RESOLVED. +===================================================================== +The shared `bindPaneShell` producer now serves relay reattach and both spawn handlers; see +[S5](./new-design-goalposts.md#s5--the-superseded-pane-fence-is-live-on-the-reattach-path--proven). + +`rememberPaneKeyForPty` (ipc/pty.ts:517-525) has two callers, both spawn (:5222, :6668). `restoreReattachedPtyRuntime` calls `runtime.registerPty` instead (ssh-relay-session.ts:2523), and `isSupersededPtyId` returns `false` for an unrecorded id by design (:284-296) — so the shipped superseded-PTY fence is INERT on the path it was built for. Collapse to one `bindPaneShell({ hostId, worktreeId, leafId, ptyId, incarnationId, relayInstanceId })` that, in one call, (a) resolves the current tab containing the leaf (refusing if none), (b) writes the durable record, (c) writes the in-memory `ptyPaneKey`/`paneKeyPtyId` fence maps using the paneKey composed from the CURRENT tab, (d) calls `registerPty`. All three paths call it and nothing else. + +===================================================================== +D8. COLLAPSE THE RESTORABLE FORK — and the offline/synchronous path +===================================================================== +`setLocalWorkspaceSession` forks on whether the INCOMING layout map is empty (persistence.ts:6450-6473): empty -> `isRestorablePtyBinding` (absence of a lease implies restorable, :6620-6631); partial -> `hasRestorableSshRemotePtyLease` (requires a live lease, :6670-6685). They disagree on the absent-record case, so whether a binding survives depends on whether a SIBLING LEAF had one. + +One predicate replaces both: RESTORABLE IFF `paneShellOwnershipByLeafId[leafId]` exists with state `attached` or `detached`. + +OFFLINE / SYNCHRONOUS ACCESS PATH (refutation B, honored exactly). `setLocalWorkspaceSession` is a synchronous `void` method invoked on every renderer publish including during quit (persistence.ts:6465, :6287, :6351). The loop at :6448-6462 already iterates `priorLayout.ptyIdsByLeafId` and therefore already holds a `leafId`, so the new predicate is a DIRECT map lookup on a top-level, main-owned, in-memory-mirrored disk record: no key composition, no lease-array scan, no `incomingHasAnyBinding` branch, no mux, no RPC, no host. It works with the host down and during quit — which is precisely what refutation B says cannot be replaced by asking the host. + +===================================================================== +D9. THE REATTACH ALGORITHM (SSH, on connect) +===================================================================== + +1. Obtain the consumer owner grant. Read `grant.relayInstanceId` (may be absent — old relay). +2. Work list = `getPtyIdsForConnection(target)` UNION `{ r.ptyId | r.hostId === 'ssh:'+target and r.state !== 'terminated' }`, every id normalized through `toRelaySshPtyId`. +3. PROCESS GATE. For each record: if both `grant.relayInstanceId` and `r.relayInstanceId` are present and differ, do not attach by that id. Set `detached`. Its shell, if any, is rediscoverable only through step 5. +4. For each remaining id, call `pty.attach { id, cols, rows }` — with NO expectedPaneKey and NO expectedTabId. + 4a. SUCCESS -> if a record exists and `response.incarnationId !== r.incarnationId`, throw (activation lease rolls back), set `detached`, no grant, no exit. Otherwise call `bindPaneShell(...)`: resolve the current tab containing `leafId`; if none, log and leave the shell running unbound (today's refusal, preserved); else write record (`attached`, `relayInstanceId = grant.relayInstanceId`), fence maps, `registerPty`. + 4b. NOT-FOUND -> if `grant.relayInstanceId` is present and equals `r.relayInstanceId`: `terminated`, retire, grant authorized (gated on E-1). Otherwise: `detached`, no grant, and route into the existing `restoreRequired = 'reattachAttemptsExhausted'` branch. No `pty:exit -1`, no `clearProviderPtyState`, no `deletePtyOwnership`. + 4c. ANY OTHER ERROR -> `detached` via the same existing branch. +5. Call `listProcesses` best-effort. If it REJECTS or admission truncates it: orphans = unknown; do nothing. If it RESOLVES: orphans = returned MINUS `(ptyId, incarnationId)` named by non-terminal records on this host. For each orphan with a reported `paneKey`, look up `parsePaneKey(paneKey)?.leafId`; if that leaf exists in the live layout and has no non-terminal record, CAS-adopt it via `bindPaneShell`. The rest are listed in the cleanup UI, never auto-killed. + +===================================================================== +D10. MIGRATION, SEQUENCING, ROLLBACK +===================================================================== +ORDER — each its own PR, each independently revertible: + +| | Step | Contents | +| --- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A | Identity-mismatch is not death | Renderer: `isProvenSshSessionGoneError` stops treating the mismatch marker as proof. Relay+main: delete the whole attach-identity apparatus (D2). Pure bug fix, no design dependency, ships FIRST; the renderer half must ship even before relays are redeployed. | +| E-0 | Stop lying about exits | Collapse the plain-not-found branch of `handlePtyReattachFailure` into the existing non-destructive `restoreRequired` branch (D3). Ships with, or immediately followed by, the D5 disconnected-pane affordance. | +| P | Single partition | D6: one accessor for every reader and writer, plus the fold migration, atomically. | +| F | Single bind producer | D7: `bindPaneShell`; makes the shipped fence live on the reattach path; introduces tab-from-layout resolution. | +| E-1 | Prove the grant executes | Production-id-shape oracle for `recoverTerminalPane`; record the actual behavior. | +| E-2 | Death rule | `relayInstanceId` on the grant + the D3 table. Conditional on E-1. | +| W | Projection | `paneKey` through relay -> `PtyProcessInfo` -> admission allowlist; incarnation-scoped subtraction; explicit-only kill (D4). | +| K | Re-key | The leaf-keyed record + D8 + D5 lifetime + `reassignSshTargetId` field rewrite + porting all lease readers. | + +E-1 CONTEXT (why E-2 is conditional). The chain `not-found -> expired -> 30s grant -> createTerminal` has one spawn-authorizing caller, `recoverTerminalPane` (src/main/runtime/orca-runtime.ts:16449-16490), gated on `getRecentExpiredSshLease(...)`, whose comparison is raw `lease.ptyId === ptyId` with no normalization (:6310). `lease.ptyId` is relay-native (persistence.ts:7184) while the graph's `pty.ptyId` is app-form `ssh:@@pty-N` (ssh-relay-session.ts:2523), so for a real SSH pane they can never be equal, and the branch also cannot be reached by a local pane (it requires an SSH lease). The covering test seeds both sides as `'pty-expired'` and registers with a NULL connectionId (orca-runtime.test.ts:1350-1370, :2932-2940) — a shape production cannot produce. Do not build grant-arbitration machinery before E-1 shows the branch executing. + +FORWARD MIGRATION AT K, per `SshRemotePtyLease`: + +- has `leafId` -> one record. `state`, `createdAt`, `updatedAt` copied VERBATIM (any surviving grace window reads `updatedAt`). `worktreeId` copied as an OPAQUE FULL STRING — the `::workspace:` folder-workspace suffix must survive, since matching is full-string equality (persistence.ts:7239, orca-runtime.ts:6308) and issue #12474 was a comparator that stripped exactly that. `ptyId` stays relay-native. `tabId` is DROPPED (D1). `relayInstanceId` absent -> the record falls to `detached` on first reconnect with no ambiguity to resolve. `incarnationId` absent on a legacy row -> treat as adoption-eligible rather than as a subtracting claim, so it cannot mask a recycled id. +- no `leafId` -> no record; recovered at first reconnect by the leaf-half `paneKey` match, or listed as an orphan. +- ALSO synthesize a record from any live layout binding (`ptyIdsByLeafId`) whose ptyId has no lease, so a bound-but-unleased pane is not silently demoted. +- Sequence AFTER the legacy numeric-leaf remap (persistence.ts:1858-1872) so every record keys on a UUID. + +PORT EVERY LEASE READER — the "write-only legacy projection" claim was FALSE and is corrected. Verified production readers of `sshRemotePtyLeases` outside persistence.ts: + +- `getRecentExpiredSshLease` (orca-runtime.ts:6295-6316) with `ptyId` UNDEFINED, feeding `hasRecentExpiredSshLeasePane` (:6318-6323) -> the headless-mobile terminal-tab filter (:5290-5299 region, the `onlyRuntimeOwnedTerminals` filter) and `hasLiveOrPersistedServeOrSshOwnedPtyBinding` (:6355-6360). These are a live VISIBILITY signal for paired mobile/HUB clients and are NOT killed by the namespace bug, because they pass `ptyId` undefined. Port: "a non-terminal record names this leaf" replaces "an `expired` lease within 30s". Visibility becomes bounded by the D5 lifetime rule instead of a timer; that behavior change is deliberate and must be covered by an oracle. +- `ssh:terminateSessions` (src/main/ipc/ssh.ts:1282-1300) enumerates leases and distinguishes owned from expired (#2626). This is the EXISTING kill affordance; the D4 orphan UI FEEDS it rather than duplicating it. It is re-expressed over records: non-terminal records plus projection entries, with kill always explicit (D4). +- `ssh:resetRelay` (ipc/ssh.ts:1370-1385) marks every live lease `expired` after force-killing the relay. Ported to: mark records `detached` (never `terminated` — the client killed the relay, which proves nothing about the shells). BEHAVIOR CHANGE, flagged for the owner: a user-initiated relay reset stops silently re-showing panes for 30s and instead shows the disconnected affordance. +- `ssh-relay-session.ts:1914, :2197` internal lease reads -> record reads. + +CROSS-VERSION READS. Pinned older builds read `sshRemotePtyLeases`. For ONE release the new writer emits a derived legacy projection of the array. It must emit RELAY-NATIVE ptyIds via `toRelaySshPtyId` — leaking app-form ids gives pinned builds leases matching nothing, a silent no-op that looks like a passing test. Because the record already stores the relay-native form, this is a copy, not a conversion. It carries no `tabId`-dependent semantics, so old builds that call `expectedIdentityForLease` simply get `null` and attach without an identity — which, after step A, is the correct behavior anyway. + +PARTIAL WRITE / ROLLBACK. Reuse the existing discipline (persistence.ts:7346-7360): snapshot, mutate in memory, `flushOrThrow`, undo on failure. One record, so this is a single-record transaction. Rollback of K: the legacy projection lets an older build boot on new state; the one-way loss is leafId-less rows, which the old build also could not act on. + +`reassignSshTargetId` (persistence.ts:7055-7086) today re-keys partitions, rewrites lease `targetId`s and drops recoveries. Because `hostId` is a FIELD and not part of the key, it only rewrites a field in place plus the reverse index; no re-keying. + +===================================================================== +D11. ORACLES — each must redden under a stated mutation +===================================================================== + +1. DETACH-TO-TAB OVER SSH, THROUGH THE REAL RELAY COMPARISON. Bind an SSH pane in tab A leaf L, run `detachTerminalPaneToTab` to tab B, disconnect, reconnect. Assert: exactly one shell on the host, no second agent resume, the pane reattaches and the ptyId lands in tab B's layout. THIS MUST BE RED BEFORE STEP A. If it is green today, the harness stubbed the relay comparison or omitted `ORCA_PANE_KEY`/`ORCA_TAB_ID` from the spawn env, and the whole detach oracle family is measuring nothing. Gate the program on it going red first. +2. Identity mismatch is not proof of death. Assert `isProvenSshSessionGoneError` is false for the mismatch string, no record reaches `terminated`, no grant. Mutation: restore `.includes(SSH_SESSION_EXPIRED)` as the sole predicate (reattach-failure-classification.ts:29). +3. Wrong-incarnation attach is abandoned, not adopted and not fatal. Relay returns a different `incarnationId` than recorded: assert no bind, no stream, `detached`, no grant, activation lease rolled back. Mutation: skip the response check. +4. Same-instance not-found is proof; different or absent is not. Two runs, recorded `relayInstanceId` equal -> `terminated` + grant; differing/absent -> `detached` + no grant. Mutation: flip the comparison. +5. Failed listing is not an empty listing. Make `listProcesses` reject, and separately make `admit` throw `agent_session_ownership_unknown`: assert zero records retired and zero orphans reported. Mutation: replace the failure with a resolved `[]` — must redden, proving the implementation does not reuse the `.catch(() => [])` shape at ipc/pty.ts:7333. +6. No fabricated exit on unknown. Drive `handlePtyReattachFailure` with a not-found and a differing `relayInstanceId`: assert `detached`, no grant, no `pty:exit -1`, and that provider state and ownership are NOT cleared. Mutation: restore the block at ssh-relay-session.ts:2641-2647. +7. Detach mid-publish does not release. Force a `session:set` flush between `setTabLayout(source)` and `setTabLayout(target)`: assert ownership is NOT released. Mutation: release on any publish missing the leaf. +8. Pane/tab delete releases. Delete a tab while its shell is live: assert the record is released, the shell IS an orphan, and no respawn is authorized. Mutation: remove the release rule — the shell must become invisible to both adopt and the cleanup UI. +9. Production id shapes in the recovery grant (E-1). Seed relay-native `pty-7`; register the runtime pty as `ssh:@@pty-7` with a NON-NULL connectionId; call `recoverTerminalPane`; assert the actual behavior. Mutation: swap either namespace. +10. E-2 false not-found. Same `relayInstanceId`, not-found caused by the wrong id form while the shell is in the Map under the correct id: assert NO `terminated` and NO grant. Mutation: skip `toRelaySshPtyId` on the attach path. +11. Projection is incarnation-scoped. Record `pty-1@incX`; restart the relay; it mints `pty-1@incY` for a different pane: assert the fresh shell IS an orphan and the stale record does NOT mask it. Mutation: subtract on ptyId alone. +12. `paneKey` survives `admit`. Mutation: remove it from the allowlist rebuild (pty-process-list-admission.ts:106-115). +13. Detached shell is still re-adopted. Seed a leafId-less lease whose relay-reported `paneKey` names a DIFFERENT tab than the live pane: assert the shell is attached exactly once and no second shell spawns. Mutation: match on full-string paneKey. This is the migration gate — green before K, re-run after. +14. Paired-client visibility survives the state rename. Record in `detached`, ptyId absent from every layout: assert `buildHeadlessMobileSessionTerminalTabs` still yields the tab. Mutation: drop the `expired`-equivalent — orca-runtime.ts:5964/:6290/:6322/:6359 must all be covered, not just the dead :16478 caller. +15. Cleanup UI reaches the user's own abandoned shells. Delete the repo row (new `repoId` on re-add) while shells run: assert they are offered for cleanup. Mutation: reinstate worktreeId-based "owned elsewhere" — must redden. +16. Nothing is killed implicitly. Any reattach/cleanup pass over foreign or unrecognized shells: assert zero kill RPCs. Mutation: auto-kill unclassified orphans. +17. One pane, one live shell across partitions. Two live shells for one `(worktree, tab, leaf)` with the binding only in `local`: assert exactly one is bound. Mutation: restore the ssh-first preference in `durablyBoundPtyIdForPane` (persistence.ts:7271). +18. P covers readers. Seed a binding only in `local`, call stable-pane resolve/retire with `connectionId` set: must find it. Mutation: revert either reader. +19. Adopt is exclusive. Two concurrent adopts of one orphan: one succeeds, the other gets a typed refusal. +20. Renderer cannot clobber ownership. Publish a stale full `WorkspaceSessionState` via `session:set`: assert the ownership map is byte-identical. +21. Offline quit path. Restorable predicate with store only and a null mux: must not throw, must not require an RPC (persistence.ts:6465). +22. Folder-workspace suffix. Two `::workspace:` instances on one path: a record for A must not reattach into B. +23. Target reassign carries ownership. `reassignSshTargetId(old, new)`: every record's `hostId` field and the reverse index follow, with no re-keying. +24. Windows/WSL relay kill. Assert the rules produce `detached`, never `terminated`, and that no UI copy asserts death. Mutation: assert-death — must redden on a Windows fixture even though it would pass on POSIX. +25. Old-relay degradation. `listProcesses` without `paneKey`, grant without `relayInstanceId`: projection non-empty, nothing killed, reattach falls back to the `getPtyIdsForConnection UNION records` work list, and the per-attach incarnation guard still rejects a recycled id. + +===================================================================== +D12. RE-TEST AGAINST THE SECTION-3 REFUTATIONS +===================================================================== +REFUTATION A LEG 1 — "a lease must name a live shell no pane owns." SURVIVES. Discovery never was the lease's job: reattach builds its list from `getPtyIdsForConnection(target) UNION` the durable rows (ssh-relay-session.ts:2211-2219), and the connect-time projection adds host-reported shells on top. Leaf-keying does not weaken this. D5's RELEASE-vs-RETIRE distinction is REQUIRED BY this leg: a record whose leaf is gone must be released (shell stays discoverable and adoptable) rather than deleted (knowledge lost) or terminated (respawn wrongly authorized). + +REFUTATION A LEG 2 — "a lease outlives the binding; expiry authorizes a 30s recreate." First clause survives exactly: `detached`/`terminated` are states on a main-owned, top-level record that persists after the layout binding is wiped. The recreate clause is PRESERVED AS A CONDITIONAL, not deleted: E-2 keeps a grant, gated on `relayInstanceId` equality, and only after E-1 shows the branch executes at production id shapes. Deleting the grant outright — proposed twice — is still refused as a simplification, because it dies to A leg 2 as written and rests on a read-not-run conclusion; it remains available as an E-1 OUTCOME. What does ship regardless is the user affordance, which covers every case a narrowed grant would drop. + +REFUTATION B — "make the host authoritative." SURVIVES, and is honored more cheaply than before. Ownership stays on local disk, top-level, main-owned, read synchronously inside `setLocalWorkspaceSession` during quit with the host down (D8). Everything this design changes is the record's KEY and WHAT MAY TRANSITION IT — never where it lives or how it is read. The host is consulted only while connected, and only for "which shells are in your Map?", "is this the same shell I bound?" (incarnation) and "are you the same process?" (relayInstanceId) — three questions it answers from data it already holds. Critically, D4's deletion of retire-on-absence is what keeps refutation B intact under load: a design that retired durable ownership because a remote listing timed out would have made the host authoritative by accident. + +REFUTATION C — a second authority architecture. Untouched, and this revision deletes more than the last: no decision table, no `hostEpoch`, no `os.uptime` math, no per-platform boot semantics, no paneKey transfer/alias layer, no attach-time identity apparatus, no orphan-ownership classifier, no second durable record. + +WHERE THE CUT IS. Plane 1 (delivery identity, credit ledger, chunk-and-ack, binary payloads, loopback WebSocket) proceeds in parallel. The cut is DURABILITY SCOPE vs EVIDENCE SOURCE, and this revision now applies that rule to all four inferences the prior round left standing: an error string means dead (deleted, D2), absence from a list means dead (deleted, D4/D5), a delivery-checkpoint flag means alive (deleted, D3), an arithmetic clock means rebooted (never built, D3). What remains on the wire is two optional fields: `relayInstanceId` on the grant and `paneKey` on the process summary. RC3 is entirely Plane 1's. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md new file mode 100644 index 00000000000..ca48bdb9e64 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/design-overview.md @@ -0,0 +1,255 @@ +# Terminal session correctness — design overview + +Plain-language explanation of what breaks, why, and what we changed. No prior +context needed. + +--- + +## 1. The cast of characters + +When you look at a terminal in Orca, five different things are lined up behind +that one rectangle on screen. + +```mermaid +flowchart LR + subgraph YOU["What you see"] + PANE["Pane
a rectangle in a tab"] + end + subgraph APP["Orca's bookkeeping"] + BIND["Binding
this pane uses that shell"] + LEASE["Lease
a note saying: we own
a remote shell"] + end + subgraph MACHINE["The actual computer"] + PTY["PTY
the shell process"] + HOST["Host
your Mac, or an SSH box"] + end + + PANE -->|bound to| BIND + BIND -->|points at| PTY + LEASE -->|claims| PTY + PTY -->|runs on| HOST +``` + +The pane is what you see. The **PTY** is the real shell process doing the work. +The **binding** is the note that connects them. On a remote machine there is also +a **lease** — a second note saying "we still own that remote shell." + +**Everything that went wrong is one of those notes pointing at the wrong thing.** + +--- + +## 2. The bug, in one picture + +You are working on an SSH machine. Your network hiccups. Orca reconnects. + +**What should happen:** your two terminals come back, same shells, same work. + +**What actually happened:** + +```mermaid +flowchart TD + A["You have 2 terminals"] --> B["Network hiccup"] + B --> C["Orca reconnects"] + C --> D["Now you have
19 terminals"] + D --> E["Hiccup again"] + E --> F["Now 20.
Most are ghosts you
never opened"] + + style D fill:#ffe6e6,stroke:#cc0000 + style F fill:#ffcccc,stroke:#cc0000 +``` + +That is a real report: **2 → 19 → 20**. Meanwhile the remote machine fills up +with shells nobody is using, until it refuses to start any more. + +A second, worse version: your AI coding agent gets **resumed twice**. Two copies +of `claude` writing into the same conversation file. One report had **five**. + +--- + +## 3. Why it happened — three root causes + +### Cause 1: the lease forgot which pane it belonged to + +Each lease is a note saying "we own remote shell X." The note _had_ a field for +"which pane," but nobody checked it. + +```mermaid +flowchart LR + subgraph BEFORE["Before — lease identified by shell only"] + P1["Pane A"] --> L1["Lease: shell-1"] + P1 -.->|reconnect makes
a new shell| L2["Lease: shell-2"] + P1 -.->|and another| L3["Lease: shell-3"] + N1["Nothing retires
the old notes"] + end + style N1 fill:#ffe6e6,stroke:#cc0000 +``` + +Every reconnect added a note and removed none. Next reconnect, Orca dutifully +restored **all** of them — one pane per stale note. + +**Fix:** a lease is now identified by _which pane_ it belongs to, not just which +shell. One pane, one live lease. Older notes are retired. + +> Retired means **marked expired, not killed**. The remote shell is deliberately +> left running — losing a note is not proof the shell died. + +### Cause 2: reconnecting could _create_ panes + +The function that says "pane A uses shell X" was also allowed to _invent_ pane A +if it didn't exist. That is correct when you open a new terminal. It is very +wrong when reconnecting. + +```mermaid +flowchart TD + R["Reconnect finds a
lease for shell-7"] --> Q{"Does its pane
still exist?"} + Q -->|"Yes"| OK["Reattach. Correct."] + Q -->|"No — before"| BAD["Invent the pane.
Ghost appears."] + Q -->|"No — after"| GOOD["Refuse. Leave the shell
running, unbound."] + + style BAD fill:#ffcccc,stroke:#cc0000 + style GOOD fill:#e6ffe6,stroke:#00aa00 +``` + +**Fix:** reattach now says _"bind only, never create."_ Spawning a new terminal +still creates, exactly as before. + +### Cause 3: a living shell was reported as dead + +The remote side has two different bad-news messages: + +| Real meaning | What it was reported as | +| ------------------------------------------------------------- | ------------------------------ | +| "This shell is gone." | `SESSION_EXPIRED` ✅ correct | +| "Shell is fine, I just need to re-establish the output pipe." | `SESSION_EXPIRED` ❌ **wrong** | + +Orca's reaction to "expired" is to start a fresh shell — and because it passes +the old session id along, the AI agent gets resumed a second time. **That is the +duplicate-agent bug.** + +**Fix:** the second case gets its own message, and respawning now requires +_positive proof_ the shell is gone. Anything unrecognised means "unknown", and +unknown never justifies starting a replacement. + +--- + +## 4. The one rule underneath all of it + +```mermaid +flowchart LR + E["Evidence"] --> A{"What does it
prove?"} + A -->|"Shell exited"| D["DEAD — cleanup allowed"] + A -->|"Can't reach it"| U["UNKNOWN"] + A -->|"Timed out"| U + A -->|"Not in a list"| U + A -->|"Tried 3 times"| U + U --> N["Do nothing destructive.
Wait. Retry. Ask."] + + style D fill:#e6ffe6,stroke:#00aa00 + style U fill:#fff4e6,stroke:#dd8800 + style N fill:#fff4e6,stroke:#dd8800 +``` + +**Unknown is not dead.** A disconnect, a timeout, an empty list, or three failed +retries tell you nothing about whether a process is alive. Every bug above is +some version of treating silence as a death certificate. + +We also made the liveness check able to _say_ "unknown". It used to return only +yes/no — so a provider whose network was down had no way to express "I can't +see, don't ask me" and answered "no", meaning dead. + +--- + +## 5. What we changed + +Six changes, ~214 lines of production code total. + +| # | Change | Stops | +| --- | ----------------------------------------------- | ----------------------------------------------- | +| 1 | Reattach binds, never creates | Ghost panes | +| 2 | Leases identified by pane | 2 → 19 → 20 growth | +| 3 | Heal already-corrupted lease data | Existing installs stuck at 20 | +| 4 | "Needs re-establishing" ≠ "expired" | Duplicate agent resume | +| 5 | Liveness can answer "unknown" | Live shells declared dead | +| 6 | One PTY's failure can't drop the shared channel | One bad pane killing every session on that host | + +### Change 6 in a picture + +```mermaid +flowchart TD + subgraph B["Before"] + X1["Pane 3 can't re-prove
its output stream"] --> X2["Give up after N tries"] + X2 --> X3["Drop the entire
connection to that host"] + X3 --> X4["Every pane, file transfer
and git command dies"] + end + subgraph A["After"] + Y1["Pane 3 can't re-prove
its output stream"] --> Y2["Park pane 3 only"] + Y2 --> Y3["Its shell keeps running.
Siblings untouched."] + end + style X4 fill:#ffcccc,stroke:#cc0000 + style Y3 fill:#e6ffe6,stroke:#00aa00 +``` + +### And a guard on every keystroke + +Orca queues your typing. If a pane switches to a different shell mid-flight, +queued keystrokes could land on the **new** shell. + +```mermaid +flowchart LR + K["Keystroke for shell-1"] --> C{"Is shell-1 still
this pane's shell?"} + C -->|"Yes"| S["Deliver"] + C -->|"No — pane moved on"| R["Refuse"] + C -->|"Pane unknown"| S2["Deliver
unknown ≠ stale"] + + style R fill:#fff4e6,stroke:#dd8800 +``` + +We did this **without changing the wire format** — Orca already keeps two lookup +tables in lock-step, so if they disagree, that _is_ proof the id is stale. No new +data is sent per keystroke. + +--- + +## 6. The approach we rejected + +A previous attempt built a **new authority architecture beside the old one**: ++60,903 lines of production code. It fixed **none** of the three root causes — +they were still present, untouched, underneath. + +```mermaid +flowchart LR + subgraph W["Rejected"] + O1["Old system
(3 bugs)"] + N1["New system
+60,903 lines"] + O1 -.->|"bugs still here"| O1 + end + subgraph R["Taken"] + O2["Old system
(3 bugs fixed)
+214 lines"] + end + style N1 fill:#ffcccc,stroke:#cc0000 + style O2 fill:#e6ffe6,stroke:#00aa00 +``` + +A second proposal of mine was also rejected in review — it would have added a +_second_ identity system beside an existing one, which is the same mistake in +miniature. Three of its factual claims turned out to be false when checked +against the code. + +**The lesson both times:** the machinery mostly existed already. The bug was that +it wasn't being _called_. + +--- + +## 7. What is still open + +| Thing | State | +| ------------------------ | -------------------------------------------------------------------------------- | +| The three root causes | Fixed | +| Duplicate agent resume | Root cause fixed; the codebase still carries 3 older patches for it | +| Orphaned remote shells | Left running on purpose. Not reachable from the UI yet — needs a recovery screen | +| The keystroke quarantine | Still needed. We proved deleting it re-opens a real `rm -rf` hazard | + +**On that last one:** there is a safety module that drops the tail of a half-typed +line after a recovery, because `echo hi; rm -rf x` can otherwise arrive as +`cho hi; rm -rf x` — the shell rejects `cho` and then happily runs `rm -rf x`. We +tried to delete it. We proved by experiment that we cannot, yet. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html new file mode 100644 index 00000000000..afe8bcbbc86 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/finalized-design.html @@ -0,0 +1,275 @@ + + + + + + Terminal Architecture — Finalized Design + + + +
+
+

Terminal Architecture — Finalized Design

+

Synthesis of the “one protocol, three routes” proposal with a control-plane track it does not cover

+
+ +
+
Adopt
Data-plane proposal
+
+1 track
Control plane (missing)
+
REFUTED
“Delete the lease”
+
~374
Lines of reconciliation
+
~390
Remoteness branches
+
+ +
+

Verdict in one paragraph

+

The proposal is correct and its root cause is real — I verified it. Adopt it. But it is a data-plane design: framing, encoding, credit windows, backpressure. The bug that started this work is a control-plane bug, and none of its thirteen steps touch the records involved. The two tracks are independent and can run in parallel. My shipped work is a stopgap on the second track, framed around a symptom rather than a cause, and most of it should be deleted once the real fix lands.

+
+ +

1 · Two planes, one system

+

Nearly every disagreement here dissolves once you separate these.

+
+
What each plane owns⤢ zoom
+

proposal covers this

proposal is silent

CONTROL PLANE — who owns what

which pane owns
which shell

what survives
a restart

what may be
destroyed

DATA PLANE — moving bytes

framing · encoding

credit windows
backpressure

reconnect & resume

13 steps

0 steps
2 → 19 → 20 survives all 13

proposal covers this

proposal is silent

CONTROL PLANE — who owns what

which pane owns
which shell

what survives
a restart

what may be
destroyed

DATA PLANE — moving bytes

framing · encoding

credit windows
backpressure

reconnect & resume

13 steps

0 steps
2 → 19 → 20 survives all 13

+
+

Verified: none of the ladder’s core files reference the lease or binding records at all. You could ship all thirteen steps and still get 19 ghost panes on the third reconnect.

+ +

2 · The data-plane root cause — verified

+

This is the strongest finding in either document, and it is correct.

+

Orca copied a well-known persistent protocol’s 13-byte frame header, and kept two of its nine message types:

+ 
export const MessageType = { Regular: 1, KeepAlive: 9 } as const
+// dropped: Control · Ack · Disconnect · ReplayRequest · Pause · Resume · None
+

The seven dropped are exactly the ones that make the protocol persistent. The ACK field that survived feeds only unackedTimestamps — a seq→timestamp map read solely by the death timer. Nothing is retained; nothing is replayed.

+
+
Why three ledgers exist⤢ zoom
+

transport gives
no delivery guarantee

SSH stack rebuilds
retain + replay

runtime RPC rebuilds
retain + replay

daemon rebuilds it
lossily on purpose

three ledgers · three sets of bugs
4,847 lines in one of them

transport gives
no delivery guarantee

SSH stack rebuilds
retain + replay

runtime RPC rebuilds
retain + replay

daemon rebuilds it
lossily on purpose

three ledgers · three sets of bugs
4,847 lines in one of them

+
+
+

This reframes the duplication as three teams correctly solving a missing primitive, not carelessness. Fix the transport and the duplication loses its reason to exist. That is a genuine root cause — the framing I shipped earlier (“identity compared with the wrong key”) named a symptom class instead.

+
+ +

3 · Your principle, confirmed twice over

+

“SSH should behave like local, plus its own reconnection logic.” The codebase already proves it — for the other routes.

+ 
function getProvider(connectionId) {
+  if (!connectionId) return localProvider          // local AND daemon — one slot
+  const provider = sshProviders.get(connectionId)  // SSH — a parallel registry
+}
+
+
Three strategies, one exception⤢ zoom
+

the exception

SSH
neither

rides the local path
wearing a connectionId tag

~390 remoteness branches
+ 25 instanceof checks

works today

daemon
substitutes

swap the provider slot
0 branches added

paired runtime
encapsulates

hidden behind a transport
mostly negative guards

the exception

SSH
neither

rides the local path
wearing a connectionId tag

~390 remoteness branches
+ 25 instanceof checks

works today

daemon
substitutes

swap the provider slot
0 branches added

paired runtime
encapsulates

hidden behind a transport
mostly negative guards

+
+

The daemon is a 3,604-line provider that adds zero branches to the shared spine. SSH gets a second registry, and the tag then leaks outward as a condition ~390 call sites re-derive independently. Your instinct was not a preference — SSH is the only route that broke a pattern the codebase already implements correctly twice.

+ +

4 · What I got wrong

+ + + + + + + + +
ClaimStatusEvidence
“Identity compared with the wrong key” is the root causeSymptomThe real cause is one layer down — a transport with no delivery guarantee, and a tag instead of a substitution
My change consolidatesInverted~310 of ~405 added lines are SSH-only special case; every new function in the largest file is lease machinery
The superseded-keystroke fence protects reconnectInert thereIts maps have two writers, both spawn paths; relay reattach never populates them, so the guard permits every reattached PTY
“Less code proves a better design”Too bluntThe right control-plane fix is roughly net-neutral in lines; the payoff is a dissolved failure class, not a smaller diff
+
+

The fence failure is the thesis in miniature

+

A guard placed on the shared spine silently does not apply to the one route that does not ride the spine. That is not a coding slip — it is the predicted failure mode of tag-based divergence, and it happened to me while I was writing the fix for it.

+
+ +

5 · The control-plane defect

+

Two questions, one record, keyed for neither.

+
+
The key inversion⤢ zoom
+

today: one record

two different questions

which shell does
this pane own?

which shells exist
that nobody owns?

lease
keyed by (target, ptyId)
pane identity is optional payload

pane uniqueness is not structural
→ bolt on supersession, ranking,
rollback, dual matchers

~374 lines of reconciliation
9 states where records disagree

today: one record

two different questions

which shell does
this pane own?

which shells exist
that nobody owns?

lease
keyed by (target, ptyId)
pane identity is optional payload

pane uniqueness is not structural
→ bolt on supersession, ranking,
rollback, dual matchers

~374 lines of reconciliation
9 states where records disagree

+
+

Because the lease is keyed by shell rather than pane, “one pane, one live shell” is not expressible — it has to be enforced. My +209 lines are that enforcement: supersession, rank comparison, deferral, rollback. They exist only to compensate for the key.

+ +

Why the obvious fixes are both wrong

+ + + + + + +
ProposalVerdictWhy
Merge the lease into the pane bindingBlockedA lease must be able to name a live shell no pane owns, and must outlive the binding — expiry deletes the binding, then authorizes recovery for 30s
Delete the lease; let the host be authoritativeRefutedSee below — tested adversarially and it failed on all five probes
+
+
Why the host cannot be the authority⤢ zoom
+

no

no

no

make the remote host
the source of truth

does it hold
the state durably?

PTY table is an in-memory Map
no persistence layer at all

can the client
read pane identity back?

listProcesses and attach never return it
attach only compares, then throws
the one RPC that returns it has 0 callers

is the host reachable
when it matters?

the decisive one:
a synchronous disk read during quit
cannot become an RPC to a host that is down

no

no

no

make the remote host
the source of truth

does it hold
the state durably?

PTY table is an in-memory Map
no persistence layer at all

can the client
read pane identity back?

listProcesses and attach never return it
attach only compares, then throws
the one RPC that returns it has 0 callers

is the host reachable
when it matters?

the decisive one:
a synchronous disk read during quit
cannot become an RPC to a host that is down

+
+

I verified all three personally. The client-side record is load-bearing precisely because it works offline.

+ +

6 · The proposed control-plane fix

+

Don’t merge the records and don’t delete one. Give each question the key that answers it.

+
+
Split by question, not by transport⤢ zoom
+

after

OWNERSHIP
keyed by (host, pane)
same key local already uses

INVENTORY
keyed by (host, ptyId)
shells with no owner

which shell does
this pane own?

which shells exist
that nobody owns?

one pane, one shell
structural, not enforced

reattach work list ·
disconnected cleanup UI

supersession, ranking, rollback,
dual matchers all become unnecessary

after

OWNERSHIP
keyed by (host, pane)
same key local already uses

INVENTORY
keyed by (host, ptyId)
shells with no owner

which shell does
this pane own?

which shells exist
that nobody owns?

one pane, one shell
structural, not enforced

reattach work list ·
disconnected cleanup UI

supersession, ranking, rollback,
dual matchers all become unnecessary

+
+

Ownership becomes the same shape for local, daemon and SSH — one record per pane per host, with transport-specific payload (lease state and timestamps for SSH, nothing extra for local). That is your principle applied at the record layer. Inventory becomes its own small list, because “a live shell nobody owns” is genuinely a different fact, not a degenerate ownership row.

+

The 30-second recovery grant survives as a terminal state on the pane-keyed record, so nothing load-bearing is lost. And the offline consumers keep reading local disk synchronously, exactly as today.

+ +

7 · Where the proposal needs correcting

+ + + + + + + + +
Claim in the proposalAssessment
“Main is no longer in the data path at all” for a phone on an SSH hostOverstated — the pivot step forwards the host’s loopback socket to a port on your machine. Pairing, E2EE and the broker remain main’s. The ledger duplication collapses; the hop does not.
Target is “one wire contract”Incomplete — no step migrates the local daemon onto it. After the final step you have two protocols, not one. There is an unstated additional step, and it is large.
“The method layer is already transport-agnostic”True, and it volunteers its own counterexample. But 191 remoteness branches sit in the renderer above that boundary — including five separate re-declarations of one predicate — and no step addresses them.
Measured encoding cost (wire and CPU)Honestly reported — it states plainly that the absolute throughput is not a bottleneck, and argues from where the cost is paid instead. Not oversold.
+ +

8 · Combined roadmap

+
+
Two independent tracks⤢ zoom
+

independent — run in parallel

CONTROL PLANE — added here

pin behaviour with
oracles that bite

re-key ownership
by pane

split out
orphan inventory

delete the
compensating code

DATA PLANE — adopt as written

see it · capability hello

unify types · ledger ·
chunk-ack · lanes

binary payloads

boring first connect

SSH becomes
a port-forward

independent — run in parallel

CONTROL PLANE — added here

pin behaviour with
oracles that bite

re-key ownership
by pane

split out
orphan inventory

delete the
compensating code

DATA PLANE — adopt as written

see it · capability hello

unify types · ledger ·
chunk-ack · lanes

binary payloads

boring first connect

SSH becomes
a port-forward

+
+

The tracks touch disjoint files, so they can proceed simultaneously. The control-plane track should land its oracles first, because re-keying a durable record is a migration with a corruption story, and the existing tests pin the current key rather than the intended behaviour.

+ +

9 · What happens to the shipped work

+ + + + + + + + + + +
ChangeNowAfter the control-plane track
Reattach binds, never createsHolds the lineKeep — correct at any key
Pane-keyed lease supersession (+209)Holds the lineDelete — the key makes it structural
Corrupted-lease healingNeeded for existing installsFolds into the migration
“Needs re-establishing” ≠ “expired”Fixes duplicate agent resumeKeep — an error-taxonomy fix, orthogonal
Three-valued livenessCorrect generalizationKeep
Superseded-keystroke fenceInert on reattachFix or withdraw — decision needed
+ +

10 · Verification status

+

Stated explicitly, because two earlier claims in this program were retracted after checking.

+ + + + + + + + + + + + +
FindingStatus
Only 2 of 9 message types kept; ACK feeds the death timer onlyPersonally verified
Daemon substitutes via the provider slot; SSH uses a parallel registryPersonally verified
Pane maps have two writers, both spawn; fence inert on reattachPersonally verified
Relay PTY table is in-memory only; identity RPC has no production callerPersonally verified
Lease consulted from a synchronous, offline disk pathPersonally verified
Method layer transport-agnostic (0 / 0 / 27 references)Personally verified
~390 branch count, LOC ratios, the 9 disagreement states, ~374 reconciliation linesAgent-reported, spot-checked
Encoding throughput measurements in the proposalNot re-measured
+ +

11 · Decisions needed

+ + + + + + + + +
#Decision
1Adopt the data-plane proposal as written? My review found no defect in its reasoning — only two overstated claims and one missing step. It is independent of everything below.
2Approve the control-plane track? Re-keying a durable record needs a migration and a corruption story. It is roughly net-neutral in lines and deletes a failure class rather than a file.
3The inert fence — populate the pane maps on the reattach path, or withdraw the guard and its test? Shipping it as-is means a test asserting a protection that does not apply where it matters.
4The open PRs — merge as a stopgap that keeps the reported bug closed, or hold them and go straight at the control-plane track?
+ +
+ Companion to report.html (evidence for the shipped work). Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
×
scroll to zoom · drag to pan · Esc to close
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql b/docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql new file mode 100644 index 00000000000..5008d103cec --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql @@ -0,0 +1,324 @@ +query TerminalSessionCorrectnessRelatedWork { + repository(owner: "stablyai", name: "orca") { + n8129: issueOrPullRequest(number: 8129) { + ...IssueDetails + ...PullRequestDetails + } + n8275: issueOrPullRequest(number: 8275) { + ...IssueDetails + ...PullRequestDetails + } + n8585: issueOrPullRequest(number: 8585) { + ...IssueDetails + ...PullRequestDetails + } + n8618: issueOrPullRequest(number: 8618) { + ...IssueDetails + ...PullRequestDetails + } + n9034: issueOrPullRequest(number: 9034) { + ...IssueDetails + ...PullRequestDetails + } + n9092: issueOrPullRequest(number: 9092) { + ...IssueDetails + ...PullRequestDetails + } + n9093: issueOrPullRequest(number: 9093) { + ...IssueDetails + ...PullRequestDetails + } + n9138: issueOrPullRequest(number: 9138) { + ...IssueDetails + ...PullRequestDetails + } + n9562: issueOrPullRequest(number: 9562) { + ...IssueDetails + ...PullRequestDetails + } + n9585: issueOrPullRequest(number: 9585) { + ...IssueDetails + ...PullRequestDetails + } + n9819: issueOrPullRequest(number: 9819) { + ...IssueDetails + ...PullRequestDetails + } + n9820: issueOrPullRequest(number: 9820) { + ...IssueDetails + ...PullRequestDetails + } + n9827: issueOrPullRequest(number: 9827) { + ...IssueDetails + ...PullRequestDetails + } + n9833: issueOrPullRequest(number: 9833) { + ...IssueDetails + ...PullRequestDetails + } + n10118: issueOrPullRequest(number: 10118) { + ...IssueDetails + ...PullRequestDetails + } + n10208: issueOrPullRequest(number: 10208) { + ...IssueDetails + ...PullRequestDetails + } + n10235: issueOrPullRequest(number: 10235) { + ...IssueDetails + ...PullRequestDetails + } + n10385: issueOrPullRequest(number: 10385) { + ...IssueDetails + ...PullRequestDetails + } + n10415: issueOrPullRequest(number: 10415) { + ...IssueDetails + ...PullRequestDetails + } + n10744: issueOrPullRequest(number: 10744) { + ...IssueDetails + ...PullRequestDetails + } + n11006: issueOrPullRequest(number: 11006) { + ...IssueDetails + ...PullRequestDetails + } + n11265: issueOrPullRequest(number: 11265) { + ...IssueDetails + ...PullRequestDetails + } + n11339: issueOrPullRequest(number: 11339) { + ...IssueDetails + ...PullRequestDetails + } + n11342: issueOrPullRequest(number: 11342) { + ...IssueDetails + ...PullRequestDetails + } + n11495: issueOrPullRequest(number: 11495) { + ...IssueDetails + ...PullRequestDetails + } + n11574: issueOrPullRequest(number: 11574) { + ...IssueDetails + ...PullRequestDetails + } + n11575: issueOrPullRequest(number: 11575) { + ...IssueDetails + ...PullRequestDetails + } + n11596: issueOrPullRequest(number: 11596) { + ...IssueDetails + ...PullRequestDetails + } + n11622: issueOrPullRequest(number: 11622) { + ...IssueDetails + ...PullRequestDetails + } + n11729: issueOrPullRequest(number: 11729) { + ...IssueDetails + ...PullRequestDetails + } + n11800: issueOrPullRequest(number: 11800) { + ...IssueDetails + ...PullRequestDetails + } + n11801: issueOrPullRequest(number: 11801) { + ...IssueDetails + ...PullRequestDetails + } + n11803: issueOrPullRequest(number: 11803) { + ...IssueDetails + ...PullRequestDetails + } + n11804: issueOrPullRequest(number: 11804) { + ...IssueDetails + ...PullRequestDetails + } + n11904: issueOrPullRequest(number: 11904) { + ...IssueDetails + ...PullRequestDetails + } + n12036: issueOrPullRequest(number: 12036) { + ...IssueDetails + ...PullRequestDetails + } + n12134: issueOrPullRequest(number: 12134) { + ...IssueDetails + ...PullRequestDetails + } + n12140: issueOrPullRequest(number: 12140) { + ...IssueDetails + ...PullRequestDetails + } + n12220: issueOrPullRequest(number: 12220) { + ...IssueDetails + ...PullRequestDetails + } + n12241: issueOrPullRequest(number: 12241) { + ...IssueDetails + ...PullRequestDetails + } + n12264: issueOrPullRequest(number: 12264) { + ...IssueDetails + ...PullRequestDetails + } + n12339: issueOrPullRequest(number: 12339) { + ...IssueDetails + ...PullRequestDetails + } + n12447: issueOrPullRequest(number: 12447) { + ...IssueDetails + ...PullRequestDetails + } + n12448: issueOrPullRequest(number: 12448) { + ...IssueDetails + ...PullRequestDetails + } + n12456: issueOrPullRequest(number: 12456) { + ...IssueDetails + ...PullRequestDetails + } + n12474: issueOrPullRequest(number: 12474) { + ...IssueDetails + ...PullRequestDetails + } + n12477: issueOrPullRequest(number: 12477) { + ...IssueDetails + ...PullRequestDetails + } + n12568: issueOrPullRequest(number: 12568) { + ...IssueDetails + ...PullRequestDetails + } + n12600: issueOrPullRequest(number: 12600) { + ...IssueDetails + ...PullRequestDetails + } + n12634: issueOrPullRequest(number: 12634) { + ...IssueDetails + ...PullRequestDetails + } + n12683: issueOrPullRequest(number: 12683) { + ...IssueDetails + ...PullRequestDetails + } + n12699: issueOrPullRequest(number: 12699) { + ...IssueDetails + ...PullRequestDetails + } + n12702: issueOrPullRequest(number: 12702) { + ...IssueDetails + ...PullRequestDetails + } + n12743: issueOrPullRequest(number: 12743) { + ...IssueDetails + ...PullRequestDetails + } + n12749: issueOrPullRequest(number: 12749) { + ...IssueDetails + ...PullRequestDetails + } + n12751: issueOrPullRequest(number: 12751) { + ...IssueDetails + ...PullRequestDetails + } + n12760: issueOrPullRequest(number: 12760) { + ...IssueDetails + ...PullRequestDetails + } + n12768: issueOrPullRequest(number: 12768) { + ...IssueDetails + ...PullRequestDetails + } + n12798: issueOrPullRequest(number: 12798) { + ...IssueDetails + ...PullRequestDetails + } + n12882: issueOrPullRequest(number: 12882) { + ...IssueDetails + ...PullRequestDetails + } + n12903: issueOrPullRequest(number: 12903) { + ...IssueDetails + ...PullRequestDetails + } + n12987: issueOrPullRequest(number: 12987) { + ...IssueDetails + ...PullRequestDetails + } + n13013: issueOrPullRequest(number: 13013) { + ...IssueDetails + ...PullRequestDetails + } + n13110: issueOrPullRequest(number: 13110) { + ...IssueDetails + ...PullRequestDetails + } + n13111: issueOrPullRequest(number: 13111) { + ...IssueDetails + ...PullRequestDetails + } + p13110: pullRequest(number: 13110) { + ...ReviewDetails + } + p13111: pullRequest(number: 13111) { + ...ReviewDetails + } + } +} + +fragment IssueDetails on Issue { + number + title + url + state + stateReason + updatedAt +} + +fragment PullRequestDetails on PullRequest { + number + title + url + state + isDraft + merged + reviewDecision + mergeable + mergeStateStatus + headRefName + baseRefName + additions + deletions + changedFiles + updatedAt + commits(last: 1) { + nodes { + commit { + oid + statusCheckRollup { + state + } + } + } + } +} + +fragment ReviewDetails on PullRequest { + reviewThreads(first: 100) { + nodes { + isResolved + isOutdated + comments(last: 1) { + nodes { + url + author { + login + } + } + } + } + } +} diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md new file mode 100644 index 00000000000..d6842d0d563 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/goalposts.md @@ -0,0 +1,683 @@ +# Goalposts and proof contract + +This file answers one question: **what exact evidence is required before an +agent may write “proven”?** + +The current status is **0/8 proven goalposts** and **2/13 proven +journeys**. Status may change only from evidence produced on the same rebased, +converged release candidate. Historical snapshot tests and independently useful +narrow PRs may be cited as partial evidence but cannot promote a row. + +## Recorded user decisions + +These amend the contract above. Only the user may add to this list. + +### D1 — G6 relaxed from strictly net-negative (2026-08-07) + +> "It's okay if we must increase LOC but try not to." + +G6's pass condition is no longer a strict inequality against the frozen +baseline. It is now: **minimise added production code, and justify every net +addition against the correctness it buys.** A net-positive total does not fail +G6 on its own; an unjustified one does. + +Why this was needed: the deletion budget the plan assumed does not exist. An +entrypoint-rooted import graph over all 20 real build entrypoints found that +51 of the 53 candidate files in `src/relay/*pty-source-*`, +`src/shared/pty-consumer-*`, and `src/main/ipc/ssh-pty-*` are reachable and +value-instantiated on live paths. Only 2 files (263 LOC) are unreachable, and 2 +of the candidates did not exist at the baseline so deleting them earns no +credit. Against roughly +1,021 LOC to offset, a strict inequality was not +reachable without deleting load-bearing code — which the governing rule forbids. + +Still binding: correctness may not be weakened to reduce line count, and a +replacement architecture added beside the old one does not earn its lines. + +## Proven journeys + +### Journey evidence not yet sufficient to promote + +**Journey 2 (daemon half).** `tests/e2e/daemon-restart-session-liveness.spec.ts`, +3 tests. Two mutations each redden exactly one clause, on macOS and Linux: +reverting three-valued `hasPty` reddens only the unknown-not-dead clause; +widening the sole-provider fallback reddens only the stale-generation clause. +The lead re-verified the first mutation independently. **Missing: physical WSL**, +which the journey names explicitly. That host is now unblocked (the distro has a +provisioned default user) and the run is outstanding. + +**Journey 12.** `tests/e2e/cross-version-wire/`, 13 tests against a real published +baseline. Reverting the restore-required publication to expiry reddens 7 of 13; +restore greens. Lead-verified. **Missing: live skew.** These are in-process wire +tests. The original ledger named live paired-runtime and SSH skew as the gap, and +an in-process decoder test does not close it. + +**Journey 4.** Sibling-pane isolation on one host reddens when +`mux.dispose('connection_lost')` is restored. **The cross-host clause cannot be +proven by mutation**: a mux belongs to one relay session per SSH target, so its +dispose cannot cross a host boundary — the cross-host test stayed green under the +mutation, which is the empirical receipt. That clause rests on +isolation-by-construction, and saying otherwise would be a false claim. + +**Journey 6.** `tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts`, +4 tests against a real Docker OpenSSH container configured `MaxSessions 1`, the +cap read back from `sshd -T` inside each test. Three of the four clauses +discriminate, clause-selectively, in single runs (the spec is not `serial`, so a +red test never reports the others as "did not run"): + +- dropping `mayCreate: false` from the reattach binding write reddens only the + authority-reconnect test, with the unowned leaf grafted into the `local` + partition; +- marking leases `terminated` instead of `detached` in `beginShutdownDetach` + reddens only the two restart tests, with a cold-spawned shell beside the + surviving one. + +**Missing: the transport-disconnect clause does not discriminate.** Four guard +removals were tried and it stayed green under all of them, including publishing +`SSH_SESSION_EXPIRED` in place of `SSH_SOURCE_RESTORE_REQUIRED` and making +`isProvenSshSessionGoneError` return true for every error. A sever that leaves +the detached relay holding its delivery record reattaches through the checkpoint +path, so no reattach failure is ever classified and the respawn-requires-proof +guards are off that path. Inducing `restoreRequired` needs a lost or mismatched +delivery record, which this fault does not produce. That clause is therefore a +forward guard and cannot promote the row. Also missing: only Docker-on-macOS was +run; the explicit-close/proven-teardown retirement clause is not covered. + +**Journey 13.** Not proven. One of ten named dimensions measured. The +`isSupersededPtyId` fence costs roughly 14ns per call, but it was measured on +lifted predicates in plain Node, not through real Electron IPC — that part is an +inference, not a measurement. + +### Correction: Journey 12's cross-version tests exercise a route production does not have + +Recorded earlier that the cross-version suite "confirms the new +`SSH_SOURCE_RESTORE_REQUIRED` token mutates nothing on an old client." That +claim is withdrawn. + +An attempt to build a LIVE two-process skew oracle established that the token +never crosses a version boundary at all. It is minted in main +(`src/main/providers/ssh-pty-provider.ts:110`) and consumed in the same app's +renderer (`pty-connection.ts:8833`, `:9080`) over Electron IPC — always one +version. The only cross-version terminal boundary, paired client to HUB runtime, +goes through `remote-runtime-pty-transport.ts`, whose sole respawn trigger is +`SSH_SESSION_EXPIRED` (line 1481), and whose reattach never reaches +`SshPtyProvider.spawn`. + +The in-process suite injects the token into `terminal.resolvePane`, which is a +pure lookup and not a route production takes. So those 13 tests are a decoder +guard, not evidence about old clients. + +Two consequences, both good: the wire-compatibility risk this branch was thought +to introduce **does not exist**, because there is no version boundary for the new +token to cross. And Journey 12 needs a different oracle entirely — one aimed at a +boundary that is genuinely cross-version. + +### Journey 6 — evidence, and why it is not promotable + +`tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts`, 4 tests, each +with its own container and Electron profile, against real OpenSSH with +`MaxSessions 1` read back from `sshd -T` rather than assumed. Remote pids are +read on the container two independent ways and must agree, each carrying its +kernel start time. + +Two mutations discriminate and are disjoint: removing `mayCreate: false` reddens +only the reconnect clause; marking leases `terminated` instead of `detached` at +quit reddens only the two restart clauses. + +**Not promotable:** the "same remote PID and exact binding survive a transport +disconnect" clause is a forward guard. Four separate guard removals left it +green. It asserts a real property, but nothing shipped is load-bearing for it, +so it cannot be claimed as proof of this branch's work. + +### Journey 3 — evidence, and why it is not promotable + +`tests/e2e/ssh-lazy-discovery-skipped-host-restart.spec.ts`, 3 tests, two real +containers, sampling sshd's own accept log and live session census 20+ times over +22 seconds with the in-use host as a positive control. + +**Not promotable:** no mutation reddens clause 3 alone. The genuine cross-host +lease-scoping guard is load-bearing, but removing it breaks the sibling host +during setup, so the failure carries no clause information. + +### Journey 2 — Daemon and physical WSL (proven 2026-08-08) + +Oracle: `tests/e2e/daemon-restart-session-liveness.spec.ts`, 3 tests, plus +`tests/e2e/helpers/daemon-shell-process-identity.ts`. The PTY leader is a real +login shell that reports `$$` back through the production write path +(`DaemonPtyRouter.write` -> daemon socket -> PTY), and that pid is resolved to a +kernel start time, so a look-alike respawn cannot pass as a survivor. + +It replaced a spec that never crossed the daemon boundary and whose successor +generation owned nothing, which made "the live successor is neither killed nor +replaced" vacuous. + +| Environment | Result | Discrimination watched | +| -------------------------------------------- | -------- | --------------------------------------------------- | +| macOS 26.3.1 arm64 | 3 passed | 2 mutations, each reddening one clause | +| Ubuntu 24.04 x86_64 native | 3 passed | same 2 mutations, same single-clause reds | +| WSL2 Ubuntu 26.04 on a physical Windows host | 3 passed | reverting three-valued `hasPty` reddens test 1 only | + +Clause-selective on all three. The spec runs `mode: 'serial'`, so a red test 1 +reports 2 and 3 as "did not run" rather than passing — selectivity was therefore +established by re-running 2 and 3 _alone under the same mutation_ and watching +them stay green, not by assuming it. + +Mutation A (`hasPty` reverted to `activeSessionIds.has(id)`) reddens only the +unknown-not-dead clause. Mutation B (widening the sole-provider fallback in +`daemon-session-owner-resolution.ts`) reddens only the stale-generation clause. +The lead independently reproduced Mutation A on macOS. + +Also confirmed on the WSL host, independently of the spec: an Orca WSL-mode +terminal now starts and returns real output. It could not start before — the +distro had no provisioned default Unix user, so every interactive `wsl.exe` +launch blocked on first-run provisioning. + +Limits, stated rather than implied: + +- The WSL run used `ELECTRON_RUN_AS_NODE=1`, which is what this oracle needs + (real daemon processes and real PTYs, no renderer). It does **not** show that + Electron's GUI starts under WSL; no display server was started and that + remains an open question for any journey needing a window. +- WSL evidence is against `d74f5ed0eae` and on Node 22 rather than the required 24. The other two platforms ran the same oracle on Node 24. +- `src/main/providers` is red on that distro for an unrelated reason: + `local-pty-shell-ready.test.ts` pins an exact OSC 133 marker count and saw 7 + splits where it expects 4. The WSL run attributed that to bash 5.3.9, but that + is **not** the cause — macOS runs the same bash 5.3.9 and the spec passes there + 67/67. The trigger is environmental to that distro, most plausibly a + system-wide bashrc contributing prompt hooks, and the underlying defect is that + the spec asserts an absolute count of markers it does not own. Outside this + journey's surface, so left for its owner rather than reinterpreted blind — but + it means "the unit suites are green on WSL" would be false. + +### Journey 1 — Local macOS, Linux, and Windows (proven 2026-08-08) + +Oracle: `tests/e2e/local-terminal-restart-binding-identity.spec.ts`. Two tests — +the same pane, full binding and OS shell process survive renderer reload and app +restart; and a stale pre-spawn session write is rejected instead of retiring the +live binding. + +Process identity is proved by the shell reporting its own pid through the +production write path, plus the kernel-reported start time, so a recycled pid +cannot pass as a survivor. + +| Platform | Result | Discrimination watched | +| --------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| macOS 26.3.1 arm64 | 2 passed | mutation reddens; restore greens | +| Ubuntu 24.04 x86_64, native | 2 passed | 4 cycles, incl. deleting `restorableBindings` reconciliation — reddens test 2 only | +| Windows 11 26200, native | 2 passed | A: forcing `shutdownDaemon()` in will-quit reddens test 1 (pid 13756 -> 8852, start times 5.4s apart). B: disabling the same reconciliation reddens test 2 only | + +Every run used an isolated TMPDIR, because the harness keys its seeded-repo +pointer on a machine-global tmpdir path and concurrent runs can both fabricate +and mask a red. + +Selectivity holds on two platforms: mutation B reddens only the stale-operation +test and leaves the restart test green, so the two clauses are independently +proved rather than jointly. + +Residual limit, stated: "every stale exact operation" is proved for the +pre-spawn session-write class. Local `pty:write`/`resize`/`signal` are fenced by +`isSupersededPtyId` with its own oracle, but that fence compares a binding, not +an incarnation — see `binding-identity-design.md`. + +## Universal proof rule + +A gate is `proven` only when both exist: + +1. the complete behavior is reachable through every named production path; and +2. the named proof exercises that behavior and its failure boundaries on the + final candidate. + +Every proof receipt must record: + +- candidate commit SHA and exact merge base; +- unfixed baseline SHA used for red/green discrimination; +- exact command or manual journey protocol; +- OS, architecture, filesystem, runtime, Git, SSH, daemon, relay, client, and + host versions relevant to that journey; +- UTC start/end time, exit status, counts, and artifact/log location; +- the production caller, persistence boundary, transport, and cleanup path + exercised; +- the expected oracle and the observed result; and +- independent reviewer identity and unresolved-finding count. + +For a new regression oracle, demonstrate that it fails for the intended reason +on the unfixed baseline and passes on the candidate. A test that passes both is +a forward guard and cannot prove the fix. + +## G0 — One design contract + +**Current status: partial.** + +G0 is proven only when: + +- every reachable production path follows one reconciled identity, host + boundary, operations, delivery, migration, compatibility, and minimal-shape + contract; +- there is one final-host authority service, thin transport adapters, one app + projection/controller, and one bounded pre-cutover legacy importer; +- no adapter or renderer owns a parallel authority state machine; +- all retained existing primitives are mapped by semantics, not similar names; +- the lost-consumer/compaction liveness gap has a safe, bounded, authenticated, + operator-reachable resolution that does not infer death from time; and +- an independent production-graph audit finds no contradictory path. + +The design document alone, a behavior contract, or focused tests do not prove +G0. + +## G1 — One final-host authority + +**Current status: partial.** + +G1 is proven only when local, daemon, WSL, direct SSH, nested SSH, paired +runtime, and remote server all resolve: + +- a stable identity minted or validated by the final PTY-owning host; +- a canonical host-local namespace for worktree, folder, and floating + workspaces; +- one exact pane-generation/PTY-incarnation binding; +- host connections keyed by final-host identity with lazy discovery; +- concurrent host isolation; and +- namespace-local admission, failure, grant, handover, and retirement. + +`connectionId`, SSH target ID, client repository ID, path spelling, or +`worktreeId` may be routing metadata. Their existing names and usage counts are +not proof that they satisfy final-host identity or namespace semantics. + +Proof requires the applicable live journeys below, including simultaneous hosts +and independent client/host updates. + +## G2 — Exact operations only + +**Current status: partial.** + +G2 is proven only when input, resize, signal, close, output, and exit are fenced +by the full binding captured before any await: + +- authority host; +- namespace; +- pane generation; +- owner/writer incarnation; +- physical PTY; +- PTY incarnation; and +- negotiated operation/source generation where applicable. + +Stale, partial, absent, timed-out, disconnected, or unknown evidence must not +affect a successor. An authoritative operation must never retry through an +ID-only provider call or a legacy mutation path. + +Proof must cover all operations across local, daemon, WSL, SSH, paired runtime, +remote server, renderer fallback, restart, concurrent replacement, and both +mixed-version directions. Store-row counts and source-text assertions are not +sufficient. + +## G3 — Durable ordered delivery + +**Current status: partial.** + +G3 is proven only when the final production design provides all observable +properties below, even if its internal mechanism differs from the preserved +construction design: + +- complete boundary snapshot before later events; +- producer held while boundary/replay is established; +- contiguous replay before reconciliation and live resume; +- durable semantic outcomes, including exit and state needed by a newly + attached consumer; +- durable idempotent main-process projection before acknowledgement; +- final-host-owned cumulative acknowledgement or an explicitly approved + equivalent with the same crash/replay guarantees; +- renderer snapshot-plus-delta observation; +- app, renderer, host, and transport restart resume; +- gap detection and resnapshot without silent omission; +- bounded memory, queues, pages, listeners, timers, and retained output; +- independent consumer retirement and safe compaction liveness; and +- no app-side duplicate cursor, settlement, receipt, or suffix-reconciliation + authority. + +Proof must include crash-before/after-ACK cuts, lost responses, disconnected +replay, gap recovery, slow/stalled consumers, retired and permanently lost +consumers, paired/remote restart, mixed versions, and scale. + +## G4 — One-way legacy cutover + +**Current status: partial.** + +G4 is proven only when each namespace performs, in order: + +1. explicit capability negotiation; +2. a brief legacy-write freeze; +3. exact non-mutating inventory; +4. a deterministic import plan; +5. validation with ambiguity kept visible and non-destructive; +6. one self-contained durable authority commit; +7. topology attachment; and +8. exact client opening through the authoritative path. + +There must be no dual writer, destructive inference, authority-to-legacy +fallback, or second durable migration catalog after cutover. Old peers remain +on an unchanged isolated legacy surface or fail before mutation. + +Proof requires crash cuts at every phase, replay from the self-contained commit, +ambiguous-row isolation, independent namespace failure, and legacy-writer and +reconciliation deletion. + +## G5 — Wire and platform compatibility + +**Current status: not started.** + +G5 is proven only when all exchanged changes follow the remote-wire +compatibility contract and the final candidate passes: + +- old client to new host; +- new client to old host; +- native macOS; +- native Linux at Ubuntu 20.04 / glibc 2.31 compatibility floor; +- native Windows; +- physical WSL, including Git Bash/`.cmd` boundaries where relevant; +- Docker OpenSSH; +- daemon; +- direct and nested SSH; +- paired runtime; +- remote server; +- git worktree; +- folder workspace; +- floating workspace; and +- drive-letter and UNC namespace paths. + +Run both skew directions independently across every changed deployment boundary: +app↔daemon, app↔SSH relay/final host, paired client↔paired runtime, remote +client↔remote server, and mobile/E2EE RPC where affected. A single in-process +codec test or one client/host pairing cannot stand in for this matrix. + +New opcodes or semantics require explicit capability negotiation. An optional +field that parses on an old peer does not by itself prove that old behavior +remains correct. + +Mocking `process.platform`, running Linux inside Docker, or passing in-process +wire unit tests does not prove the corresponding native or live-skew row. + +## G6 — Simpler, minimised production code + +**Current status: partial; structural clauses remain open.** + +Decision D1 replaces the original strict inequality: the final integrated +program must minimise production source and justify every net addition against +the correctness it buys. + +The default baseline is +`5ed45739e94bdf6460364e033bfcec9b32c0b42a`, the base recorded by GitHub for +PR #12600. This broader program subsumes #12600. Changing the baseline requires +an explicit user decision recorded before more implementation begins. + +G6 is proven only when: + +- aggregate program-attributable production source net LOC is **minimised and + every net addition justified** against the frozen baseline — see decision D1 + under "Recorded user decisions", which replaced the original strict + inequality once the assumed deletion budget was shown not to exist; +- every program-attributable prerequisite merged after the baseline and every + stacked PR is included, even if a later rebase places it in `main`; +- overlapping changes are recomputed from the frozen baseline to the final tree + so additions and later deletions are not double-counted; +- unrelated upstream or user deletions cannot offset program additions; +- production, test, documentation, CI/runner, generated, and vendored changes + are reported separately; +- every new production module is reachable from a real entrypoint; +- no test fixture remains under production compilation; +- there is one identity comparison, transition implementation, exact-operation + client, mutation admission path, and delivery state machine; +- no re-export shim or one-type module exists solely to preserve construction + layering; +- no superseded quarantine, sliding-window, retry-verdict, reconciliation, + duplicate cursor, receipt ledger, legacy writer, or migration bridge remains + reachable after cutover; and +- an independent reachability and duplicate-state-machine audit is clean. + +Tests or docs cannot offset positive production LOC. A smaller narrow PR is not +proof if the final stack remains net positive. Deleting correctness or platform +coverage to hit the number is prohibited. + +Classify by behavior, not directory name: shipped runtime code, migration code, +and build-time code that enforces a shipped artifact invariant are production; +test-only runners and fixtures are tests even when misplaced, and their presence +under production compilation independently fails this gate. Publish the final +file-by-file classification so the count cannot be moved between buckets. + +### G6 clause assessment (2026-08-08) + +Checked against the current branch rather than assumed: + +| Clause | State | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| aggregate net production LOC minimised and justified (D1) | +228, each addition justified in its commit; 201 LOC deleted after verifying unreachability from all 20 build entrypoints | +| every new production module reachable from a real entrypoint | holds — no new unreachable module was added | +| no test fixture under production compilation | not audited on this branch | +| one identity comparison / transition / exact-op client / admission path / delivery state machine | **fails** — several remain, and this is the clause the rejected binding proposal would have made worse by adding a second identity comparison beside `resolveStablePaneOwner` | +| no re-export shim or one-type module for layering | holds for this branch's additions | +| **no superseded quarantine, sliding-window, retry-verdict, reconciliation, duplicate cursor, receipt ledger, legacy writer, or migration bridge reachable** | **fails** — `terminal-input-quarantine.ts` is imported by `pty-connection.ts:98` and `terminal-pane-recovery.ts:6` | +| independent reachability and duplicate-state-machine audit clean | not run | + +On the quarantine specifically: it is **not** subsumed by the superseded-PTY +fence, and claiming otherwise would be wrong. The fence refuses writes aimed at +a stale ptyId. The quarantine guards the user's _subsequent_ typing — the tail +of a half-typed line — landing on the successor shell under its current and +correct ptyId, which the fence never sees. + +Deleting it therefore requires contract property B: a _different_ shell surfaces +unresolved and never silently receives the pane, at which point there is no +mangled tail to suppress. That is a behaviour change to the recovery path, not a +deletion, and an earlier attempt to shortcut it was rejected for reintroducing +the `echo hi; rm -rf x` hazard by disarming per-pane state that is keyed per-tab. + +G6 is therefore not promotable, and its blocking clauses are structural rather +than arithmetic. + +### The input quarantine is load-bearing, not superseded (2026-08-08) + +G6's clause lists "no superseded quarantine ... remains reachable" and +`terminal-input-quarantine.ts` was assumed to be one. It is not, and the +evidence is direct. + +**The hazard is live.** Disabling the single call site in `pty-connection.ts` +and running the pane's own oracle reproduces it verbatim, lead-verified: + + AssertionError: expected "vi.fn()" to not be called with arguments: + [ 'cho hi; rm -rf x' ] + +Deleting the module without a replacement re-opens `rm -rf` execution. + +**Contract property B was costed by building it, not estimated.** Threading the +incarnation to the renderer compiles at **+26 production LOC**, and completing it +(publishing an incarnation from `resolveTerminalPane`, plus a new exported reader +for the module-private `ptyIncarnationById` map) is about **+33**. The +cross-remount state the comparison needs must outlive the destroyed pane, so it +becomes a module of roughly the size of the one being deleted — the two in-tree +precedents are 88 and 86 lines. Adding a renderer-side identity comparison would +also worsen G6's already-failing "one identity comparison" clause. Floor: about +**+140 production LOC to delete 88**. + +**And the route is not uniformly available.** `RuntimeTerminalCreate` and +`RuntimeTerminalResolvePane` carry no `incarnationId`; remote connect results are +built without one; and the transport latches `resolvePaneUnavailable` when a host +answers `method_not_found`. Mixed client/host versions are the normal state, and +an optional field does not make an old host publish it. So a paired client reads +_unknown_ — which under this program's own governing rule is not proof of "same +shell". Requiring proof makes every remote reattach surface unresolved; not +requiring it needs a fallback, and the only correct fallback is this quarantine. +Either way the module stays reachable. + +`endpointReplaced` is also routine rather than rare — a daemon death remounts +every live pane — so property B would convert each into a manual per-pane +reconnect, which is the coverage deletion G6 explicitly prohibits. + +**Consequence:** this clause of G6 cannot be closed by deletion. Either the +clause is amended to recognise the module as load-bearing, or the program accepts +a net-positive change to replace it with something weaker on remote hosts. That +is a user decision (call it D5), not one to assume. + +### G6 clause: test fixtures under production compilation (audited 2026-08-08) + +Audited by importer rather than filename, then checked against what the build +actually emits. **The clause is already satisfied on the meaning that matters, +and cannot be closed by moving files on the other meaning.** + +36 candidates; 4 have genuine production importers and are correctly placed. The +other 32 (~3,300 LOC) have zero non-test importers. + +**They do not ship.** Spot-checked the emitted bundle for +`terminal-restore-parity`, `ipcEventsTestHarness`, `storeTestHelpers` and +`sshRelayNativeDepsInstallFixture`: none appears in `out/`. Rollup drops them +because no production entrypoint reaches them. Under "compiles into the shipped +product", this clause holds today. + +**Moving them would close nothing under the other reading.** +`config/tsconfig.node.json` and `config/tsconfig.tc.web.json` declare `include` +globs such as `../src/main/**/*` with **no `exclude` at all**. A `__tests__/` +directory is matched by that glob exactly as any other path is, and so is every +`*.test.ts` file in the repo. Relocating 32 fixtures would not remove one file +from typecheck scope. + +A sweep was started and stopped once this was verified, rather than landing 32 +moves across areas this program does not own for no measurable gain. + +If the intent behind the clause is that typecheck scope should exclude test code, +that is a repo-wide tsconfig change affecting every test file — different work +with a different owner, and it should be stated as such rather than pursued by +relocating fixtures. + +One genuine defect was found and fixed while auditing: +`terminal-pane/xterm-bypass-event-fixture.ts` and +`terminal-pane/__fixtures__/xterm-bypass-event.ts` were byte-identical apart from +an import path, and the `__fixtures__` copy had zero importers — a half-finished +move left in place. The dead copy is deleted and the live one completed its move. + +## G7 — No regression and reviewable comprehensive change + +**Current status: not started.** + +G7 is proven only on the rebased, converged candidate after G0–G6 and all +thirteen journeys are proven. It requires: + +- correctness and security gates; +- A/B input latency and output throughput; +- backpressure and bounded-memory results; +- renderer/app/daemon/relay restore and startup results; +- large-pane, long-session, and multi-host scale results; +- native packaging/startup on macOS, Linux, and Windows; +- WSL, Docker SSH, paired, remote, folder, floating, worktree, drive, and UNC + coverage; +- both live mixed-version directions; +- final categorized LOC census; +- independent repository review with no unresolved P0–P2 findings; +- release-readiness review with no unresolved correctness, security, + compatibility, or performance findings; and +- a detailed comprehensive PR whose claims match the receipts. + +Before implementation, record a performance protocol that makes “no +regression” falsifiable: + +- fixed candidate and baseline builds, hardware, OS, power mode, network shape, + pane/session population, payloads, and background-load policy; +- warm-up policy, randomized A/B order, sample count, raw-data location, and + statistical method; +- input latency, output throughput, memory, backpressure, restore, startup, and + large-pane metrics with directionality; +- deterministic ceilings for writes, scans, queues, listeners, timers, and + allocations on hot paths; and +- a predeclared equivalence/no-regression bound no larger than measured baseline + noise. A nonzero bound handles measurement noise; it is not permission for a + known slowdown and requires explicit user approval. + +Unless an independently reviewed protocol justifies another count, use at least +five warm-up trials and thirty measured trials per latency/startup/restore +configuration, retain raw samples, and report confidence intervals. Throughput, +backpressure, and memory tests must also include a fixed-duration steady-state +run and a leak-slope result. Choose all workloads and thresholds before looking +at candidate results. + +Green CI, thousands of tests, mergeability, or “reviewable for what shipped” do +not prove G7. + +## Thirteen required production journeys + +Every row is currently **not proven**. + +The issue-to-journey matrix in +[`related-open-work.md`](./related-open-work.md#mandatory-issue-to-journey-matrix) +is part of these journeys, not optional context. Every bound incident needs a +red-on-baseline/green-on-candidate oracle, or explicit evidence plus user +acceptance that it is unrelated. + +| # | Journey | Required oracle | +| --: | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Local macOS, Linux, and Windows | The same pane, full binding, and OS process survive renderer and app restart; every stale exact operation is rejected. Run natively on all three OSes. | +| 2 | Daemon and physical WSL | The same PTY survives client and daemon reconnect/restart boundaries; generation skew fails closed without killing or replacing a live successor. | +| 3 | Lazy discovery and skipped-host restart | An unused host is not probed eagerly. After a restart that skipped it, lazy rediscovery restores only that host's sessions and never adopts host-current or sibling-host state. | +| 4 | Concurrent multi-host connections | At least two distinct final-host connections operate simultaneously. One host's disconnect, CAS, timeout, or failure cannot affect the other. | +| 5 | Namespace-partial admission failure | One namespace on a multiplexed connection fails challenge/CAS/grant publication while another commits. Only the failed host-plus-namespace is fenced. | +| 6 | Docker OpenSSH with `MaxSessions=1` | The same remote PID and exact binding survive disconnect and client restart. Authority restart imports exactly or exposes unresolved recovery without creating, killing, or adopting. Explicit close or exact proven teardown durably retires the lease and process; unknown ownership stays visible and recoverable rather than being killed. Count actual remote processes, panes, tabs, bindings, leases, and session-cap slots before and after a settle interval. | +| 7 | Two independent Docker SSH hosts | Two simultaneous final hosts keep endpoint credentials, principals, namespaces, sessions, cursors, failures, and cleanup completely isolated. | +| 8 | Paired client and remote server | The final host remains authoritative across independently updated peers, pairing reconnect, client restart, and remote-runtime restart. | +| 9 | Worktree, folder, floating, drive, and UNC namespaces | Each resolves the same stable host-local namespace across spelling/restart changes, without using client repository ID or target ID as identity. | +| 10 | Stable proof and exact retry | One bounded device proof identity admits fresh process/session nonces. A lost response retries the exact challenge/request; changed, replayed, cross-host, cross-namespace, or host-current state is rejected. | +| 11 | Identity reset and re-enrollment | Crash-resumable host retirement, relay revoke acknowledgement, transport closure, local credential removal, and atomic successor publication occur in order. Offline and old peers remain explicitly pending. | +| 12 | Mixed versions in both directions | No unknown opcode or ungranted publication mutates state. Unsupported challenge, grant, delivery, or operation semantics stay on isolated legacy behavior or fail before mutation. | +| 13 | Performance and scale | Under the predeclared protocol above, input/output latency, throughput, backpressure, memory, reconnect/restore, startup, large-pane, long-session, and multi-host ceilings show no regression against the fixed baseline. Raw samples, confidence intervals, deterministic counters, and leak slopes satisfy their predeclared bounds. | + +## Cross-cutting correctness cases + +Every relevant journey must exercise: + +- stale, missing, unknown, rejected, timed-out, and disconnected evidence; +- concurrent replacement and sibling-host/namespace isolation; +- lost request and lost response; +- cancellation and partial setup cleanup; +- crash immediately before and after every durable boundary; +- restart from disk rather than process-memory state; +- duplicate and out-of-order delivery; +- gaps, overflow, and slow/stalled consumers; +- explicit close versus detach; +- eventual exact retirement after explicit close or proven teardown while + uncertain ownership remains intact and visible; +- legacy data with missing optional fields; +- relay/daemon incarnation reuse; +- sleep/resume and clock movement where timers schedule retries; +- exact remote process/PID census, not only persisted-row state; and +- cleanup that requires positive identity proof. + +## Forbidden proof substitutions + +- A mock is not a native-platform journey. +- A source grep is not a production call-path journey. +- A row marked `expired` is not proof that its remote process exited. +- Optional pane metadata plus an O(n) repair scan is not structural uniqueness. +- An empty list from an unavailable provider is not proof of absence. +- A timer or retry budget is not proof of death. +- A broad test count is not a correctness oracle. +- A test that passes unfixed code is not evidence that a change fixed the bug. +- An open or green PR is not shipped behavior. +- Historical construction receipts are not proof for a rebased candidate. +- Prior art or an uncited LOC comparison cannot delete a requirement. + +## Promotion template + +Before changing any row to `proven`, add a receipt containing: + +```text +Gate or journey: +Candidate SHA: +Merge-base SHA: +Unfixed/red SHA: +Production path exercised: +Exact command or manual protocol: +Environment and versions: +Expected oracle: +Red result: +Green result: +UTC start/end: +Artifact/log: +Independent reviewer: +Unresolved P0/P1/P2: +LOC/performance impact: +``` + +If any field is missing, the row remains partial or not started. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md new file mode 100644 index 00000000000..fce53501a7c --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md @@ -0,0 +1,386 @@ +# New design — goalposts + +Tracks the design approved on 2026-08-09 (`design-explained.html`, detail in +`counsel-design.html`). Updated as work lands. Progress in every status update +is reported against this file. + +## How to read a status + +| Status | Means | +| --- | --- | +| **PROVEN** | An oracle asserts the behaviour, removing the production guard **reddens** that oracle, and the mutation was verified to actually land before the result was believed. | +| **ORACLE RED** | The oracle is written and currently fails for the right reason. The production change is not written yet. This is the intended pre-implementation state. | +| **NOT STARTED** | No oracle. | +| **BLOCKED** | Waiting on another goalpost or an owner decision. | + +A green test is **not** a status. This program shipped three guards that passed +their tests while sitting off the route production takes; "PROVEN" exists to +make that impossible to claim by accident. + +--- + +## Scoreboard + +| | Count | +| --- | --- | +| Step goalposts proven | **7 of 7** (one deleted as unreachable) | +| Global goalposts proven | **5 of 6** — G1 missed, see below | +| Scope removed by evidence | S7 (death rule) — never built | +| Oracle clauses green | 26 | +| Oracle clauses red (awaiting implementation) | 0 | +| Net production lines, correctness work | **+13** | +| Net production lines, incl. the approved affordance | **+83** | +| Net test lines | +2,578 | + +**G1 is not met, and that is reported rather than smoothed over.** The design's +net-negative target assumed the remaining steps were collapses. Three were +(S3 −21, S8 −32, and the relay half of S5), but two genuinely add code: the +single `bindPaneShell` producer (+60, since one function replaces three +divergent call sites without those sites shrinking much) and the D1 +disconnected-pane affordance (+70), which is a new product surface the owner +approved — a feature, not a refactor. Excluding the affordance the correctness +work is +13, i.e. roughly a wash. See "G1" below for the full accounting. + +--- + +## Step goalposts + +### S1 — An identity mismatch is never read as death · **PROVEN** + +*Guarantee.* The relay reports a pane-identity mismatch by saying the pty was +not found. It found it — comparing is how it noticed. That must never reach a +respawn decision as proof of death. + +- Oracle: `src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts`, + `src/renderer/src/components/terminal-pane/reattach-failure-classification.test.ts` +- Clauses: 7 + 3 +- Mutations proven: restoring the destructive wrap reddens 2 clauses; deleting + the classifier guard reddens 1 — clause-selective, each proven separately. +- Landed: `e2524b0472f` + +### S2 — Pane identity is not sent on reattach · **PROVEN** + +*Guarantee.* Moving a pane to another tab must never make its terminal +unreachable. + +- Oracle: same file, `reattach does not ask the relay to police pane identity` (2 clauses) +- Also inverted an existing test that pinned the removed behaviour, so the new + intent stays covered rather than silently dropped. +- Works against **already-deployed relays**: the relay's comparison is + presence-guarded, so not sending the fields disarms it everywhere. No wire + change, no redeploy. +- Landed: `c51be8072ba` + +### S3 — A failed reattach never fabricates an exit · **PROVEN** + +*Guarantee.* On an unproven not-found, the pane is not told the program exited, +ownership is not deleted, provider state is not cleared, and the lease is not +expired. The case routes into the non-destructive recovery branch that already +exists. + +- Oracle: `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts` — 8 green +- Mutation proven: restoring the destructive block reddens 6 of 8 clauses; the + 2 producer pins stay green, so the mutation is clause-selective. +- Two tests pinned the deleted premise ("attach verifies liveness before + answering not-found") and were **inverted**, not patched. +- Shipped with the disconnected-pane affordance below, since panes now stay + visible instead of silently respawning. +- Landed: `c19f1b386b4` (−21 production lines) + +### S4 — One partition per (target, pane) · **PROVEN** + +*Guarantee.* A pane that re-leases under a new relay id across reconnects ends +with exactly **one** live claim; the predecessor is superseded, not left live. + +- Oracle: `src/main/ssh-pane-binding-partition.test.ts` — 7 green (one clause added) +- The highest-value goalpost: the mechanism behind the reported 2 → 19 → 20. +- Mutation proven as a **2×2**, because the two edits mask each other: + + | reader | load fold | result | + | --- | --- | --- | + | hedge (original) | off | 6 red — baseline | + | hedge | off | 4 red — the 2 source-text clauses now green | + | local-only | off | 1 red — the reader carries 3 clauses | + | hedge | **on** | **all green** ← the fold masks the reader | + | local-only | on | all green | + + That fourth row is why an **eighth clause was added**: with the fold shipping, + the fold erases the divergent copy at boot, so the reader guard would have + shipped unproven — precisely the failure mode G5 exists to catch. The new + clause rewrites `ssh:` mid-session (orphan adoption still writes + there) and reddens when the hedge is restored, pinning the reader on its own. +- Five clauses in `ipc/pty.test.ts` pinned the two-partition shape and were + **inverted** to the single home, each keeping an arity check so a re-added + partition argument fails loudly. +- Side effect found while verifying: the renderer never hydrated the `ssh:*` + partition (`listKnownRuntimeHostIds` filters to `runtime:*`), so the Issue + #217 force-quit binding protection had never worked for SSH panes. It does now. +- Landed: `2733c59879b` (+9 production lines — the call sites shrank; the fold + is new state repair) + +### S5 — The superseded-pane fence is live on the reattach path · **PROVEN** + +*Guarantee.* After a relay-driven reattach binds a pane, a stale write aimed at +the superseded predecessor is **refused**. It used to be permitted, because the +fence's bookkeeping was only written by spawn. + +- Oracle: `src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts` — 4 green + (one clause added) +- Collapsed to one `bindPaneShell` producer used by the relay reattach and both + spawn handlers. Error policy stays at the call sites because it genuinely + differs: a caller that just created a shell must clean it up on a failed + durable write; a caller that merely reattached must not detach anything. +- Mutation proven, both sub-guards isolated: + - drop the `rememberPaneKeyForPty` call → 3 clauses redden + - prefer the stored `tabId` over the live layout → 1 clause reddens +- That second clause is **new**. Every pre-existing clause used one tabId on + both sides, so a producer that simply forwarded `lease.tabId` would have gone + green and shipped the moved-pane bug unnoticed. Only the leaf half of a pane + key is remint-stable; `detachTerminalPaneToTab` moves a live pane and its PTY. +- Two source-text clauses were **strengthened**: they used to require the relay + to hold a `persistPtyBinding` call and merely forbade an ssh-partition + argument. The relay now has none, so they assert **zero** direct binding + writes there — a second bind producer is exactly the defect being removed. +- Repaired a latent false green: the "persistence fails" case in + `ssh-relay-session-reconnect-incarnation` was passing because a missing mock + made the call throw a TypeError that happened to emit the asserted + `console.error`. The failure is now injected at the producer. +- Landed: `0f409055d3f` (+60 production lines — the one step that grows) + +### S6 — Settle whether the 30s recovery grant executes at all · **PROVEN — it is dead code** + +*Result.* It cannot execute for a real SSH pane. Verified personally: + +- The lease stores a **relay-native** pty id, normalized on write + (`persistence.ts:7184`, comment: "app ids are global"). +- The caller passes the **app-form** id (`orca-runtime.ts:16478` ← `toAppSshPtyId`). +- The comparison is raw `lease.ptyId === ptyId` (`orca-runtime.ts:6310`) with no + normalization, so the two forms can never be equal. +- The branch is also unreachable for a local pane, since it requires an SSH lease. +- Its covering test seeds both sides as the same literal with a null + connectionId — a shape production cannot produce. + +### S7 — The death rule · **DELETED on S6's evidence** + +Not built. The arbitration machinery existed to referee a branch that never +executes. Per the design: "if E-1 shows the grant branch unreachable, E-2 is +deleted and the marker simply never fires, which is the safe end state." + +### S8 — Remove the dead recovery-grant path · **PROVEN** + +*Guarantee.* Deleting it changes no behaviour, because it has none. + +- Oracle: `src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts`, written + and green **before** the deletion, so inertness is demonstrated rather than assumed. +- It mints **both** id forms from the production helpers rather than as two + hand-typed literals, so it tracks the real namespace split instead of + restating it, and seeds a lease qualifying on every other predicate (state, + worktree, tab, leaf, grace window). Anti-vacuity clauses pin that control + actually reaches the gate rather than bailing out earlier. +- Mutation proven **before** deleting: normalizing the `lease.ptyId === ptyId` + comparison makes the grant fire and reddens the oracle. That is the exact fix + someone would reach for, so the oracle is pinned to unreachability, not to the + throw. +- Deleted: the grant tail, the `terminalPaneRecoveryByIdentity` dedup map (whose + only consumer was the grant), and the dead `ptyId` parameter. +- **Not** deleted, because over-deleting here breaks users: `getRecentExpiredSshLease`, + `hasRecentExpiredSshLeasePane` and `SSH_PANE_RECOVERY_GRACE_MS` all stay. Their + other two callers pass `ptyId` undefined, which short-circuits the broken + comparison — those are live and feed headless-mobile terminal-tab visibility. +- Also **not** done: deleting only the gate while keeping the spawn. That would + have granted a respawn to every disconnected pane — a change in the dangerous + direction. The refusal is what stays. +- Three tests pinned the grant and were **inverted**; a fourth is now + tautological and carries a comment saying so rather than being left hollow. +- Honest limit, recorded in the oracle: unreachable **by construction** for SSH + panes; for a local pane, unreachable only up to a random-UUID collision. +- Landed: `7d44315437a` (−32 production lines) + +### Product affordance (D1) — **SHIPPED with S3** + +A pane that cannot be verified renders as disconnected with two explicit actions +— "Try again" and "Start a new terminal" — instead of silently respawning. + +- No new IPC channel: the silent-respawn decision was always renderer-local. + Both actions are things the code already did, moved behind a click. +- Copy constraint enforced **as an oracle**, not as a review note: the rendered + text must match no death verb and show no wire token. STYLEGUIDE.md:236 already + forbids result verbs without result data, and a failed attach is not result data. +- `TerminalRemoteRuntimeReconnectBanner` → `TerminalPaneDisconnectedBanner`; it + now serves any transport. Existing i18n key strings kept verbatim so no shipped + translation breaks; the SSH copy is additive. +- Landed: `5e1b57bb2f9` (+70 production lines) + +--- + +## Global goalposts + +### G1 — Net production code is negative · **NOT MET (+83 this phase)** + +Counted from the pre-work merge base, production only. Reported as a miss rather +than reframed until it passes. + +| Step | Net production | +| --- | --- | +| S3 — collapse the fabricated exit | **−21** | +| S4 — one partition | +9 | +| S5 — one bind producer | **+60** | +| S8 — delete the dead grant | **−32** | +| D1 affordance (new product surface) | **+70** | +| Banner rename (pure move, no behaviour) | +36 of the above, not real growth | +| **Correctness work only (S3–S8)** | **+13** | +| **Total this phase** | **+83** | + +Why it missed. The target assumed the remaining steps were collapses. Three were. +Two were not, for reasons worth stating: + +- **S5 (+60).** One producer replacing three divergent call sites only shrinks + the code if those sites were duplicating the whole bind. They were not — they + each did *part* of it differently, which is exactly the defect. The producer is + net-new; the call sites shrank by 7 lines. Collapsing the two spawn handlers' + surrounding logic as well would buy the difference back, but it would reorder + side effects that existing ordering assertions pin, so it was left alone. +- **D1 affordance (+70).** A new user-facing surface the owner approved. Features + add lines; counting it against a refactor budget would be the wrong pressure — + the way to make this number go negative would be to delete the affordance. + +Where the remaining slack is, if the goal is to be met later: steps P and K of +the design (the leaf-keyed record replacing `SshRemotePtyLease`, and the lease +readers ported onto it) are the deletions this phase deferred. + +### G2 — No redeploy required for the user-visible fixes · **PROVEN for S1–S8** + +Fixes must work against relays already installed on people's hosts. Every step +that landed is client-side only: + +- S1, S2 — proven previously. +- S3 — deletes client-side reactions to a relay message; the relay is unchanged. +- S4 — client-local persistence layout only; nothing crosses the wire. +- S5 — client-local in-memory fence maps and the durable binding. +- S8 — deletes a client-side branch. +- D1 — renderer only. + +**No relay redeploy is required for any of it**, which matters because the +reported failure is happening on hosts people have already deployed. + +### G3 — Wire compatibility · **SATISFIED — nothing on the wire changed** + +Clients and hosts update independently. This phase added **no** RPC parameter, +no stream opcode and no published field, so there is nothing to negotiate. The +orphan projection that would have added one optional field belongs to step W, +which is not in this phase. + +One wire-adjacent behaviour change is worth naming for reviewers: `terminal.recoverPane` +(RPC) now always refuses for a disconnected pane. The method still exists and its +signature is unchanged, so older clients get a refusal rather than a +`method_not_found` — and the renderer already handles refusal, because that is +what the branch did in practice anyway. + +### G4 — Cross-platform · **SATISFIED by construction** + +macOS, Linux, Windows, WSL. No oracle added in this phase shells out, and none +uses `echo $$` or `ps` — they assert against persisted state, in-memory maps and +rendered text. No rule that landed concludes anything from a relay restart or +from whether a shell survives a hard relay death, which is the Windows-divergent +question the design deliberately refuses to depend on. The affordance copy holds +on every platform precisely because it never claims the shell died. + +### G5 — Every guard is proven live on the production route · **PROVEN for S1–S8** + +For each new guard, an oracle must redden when the **producer** is removed — not +only when the guard itself is removed. + +This exists because it had failed three times here: an inert `mayCreate`, a +keystroke fence inert on reattach, and a respawn gate on a minority path. S5 is +the remediation of the second. + +**It nearly failed a fourth time in this phase, and the rule caught it.** S4's +reader guard (`durablyBoundPtyIdForPane` reading one partition) initially showed +*no* mutation response: restoring the ssh-first hedge left all six clauses green, +because the load fold had already erased the divergent copy at boot. The guard +was correct and would have shipped unproven. The fix was an added clause that +rewrites the ssh partition **mid-session**, which reddens on the hedge and pins +the reader independently of the fold. + +Two further near-misses found and closed the same way: +- S5's clauses all used one tabId on both sides, so "compose the paneKey from + the current tab" was unpinned — a producer forwarding `lease.tabId` would have + gone green. A moved-pane clause now pins it. +- A case in `ssh-relay-session-reconnect-incarnation` was a **latent false + green**: it passed because a missing mock threw a TypeError that happened to + emit the `console.error` it asserted. Now injected at the producer. + +### G6 — No `max-lines` disables · **HOLDING** + +Per AGENTS.md. No per-file bumps either. + +--- + +## Definition of done + +| | Criterion | Status | +| --- | --- | --- | +| 1 | S1–S5 and S7 PROVEN, or S7 deleted on S6's evidence | **MET** — S1–S6 and S8 proven; S7 deleted on S6's evidence | +| 2 | G1–G6 satisfied, each with evidence recorded here | **PARTIAL** — G2–G6 met; **G1 missed at +83**, accounted for above | +| 3 | The reported failures covered by an oracle that reddens without the fix | **MET.** Pane cardinality: S4's ten-reconnect clause, plus an E2E spec driving three real reconnects against a Docker relay (passes). Duplicate resume: the classification clauses, plus an E2E spec where the **host itself** records every shell launch (passes). Writing the second one is what exposed that RC2 was still live — see below | +| 4 | No guard shipped without a producer-side mutation proving it is reachable | **MET** — and it caught one guard that would otherwise have shipped unproven (see G5) | + +### Found after the goalposts were met + +Two defects surfaced *after* every step goalpost was proven, and both were invisible to the unit +oracles that had already gone green. Recorded because they are the most useful evidence in this +phase about where these oracles stop reaching. + +**RC2 survived S3 — found by writing the E2E test.** `7cd7fef927f`. +The reported duplicate agent resume was not fixed. `reattachSshPtySession` maps every relay +not-found to `SSH_SESSION_EXPIRED`, and `isProvenSshSessionGoneError` returned true for it, so the +renderer's reattach arm still respawned. The E2E harness produced the shape directly: a stalled +relay is superseded by a fresh one with no memory of `pty-1`, while the old shells keep running +under the stopped process. The new relay answers not-found for a live shell. + +A not-found proves an exit only if the relay process that minted the pty is the one answering — +design D3 row 2, gated on `relayInstanceId`, which step E-2 never built. `SSH_SESSION_EXPIRED` is +not independent evidence: its only producer is that same mapping, and the token's own doc comment +claimed "the host proved the session is gone", which it never did. + +So `isProvenSshSessionGoneError` returns false. This is also what makes the D1 affordance +load-bearing rather than near-unreachable. The respawn tails are deliberately kept — the design +preserves the grant as a conditional for E-2 — with a clause pinning that nothing reaches them. + +**The fold left half a fence behind — found by adversarial review.** `994733d8b1a`. +`persistPtyBinding` writes the binding and its incarnation into the same partition, and the +incarnation is what its CAS compares. The fold moved only `ptyIdsByLeafId`, so after upgrade the +guard stayed in the partition nothing reads and the CAS compared `undefined` against `undefined`. +Not data loss and not a wrong-shell bind, but a guard silently weakened by a migration is the exact +shape this program keeps finding. + +### Residuals, stated rather than closed quietly + +- **Orphan adoption still writes the `ssh:` partition.** + `adoptTerminalOrphansFromInventory` (via `tryGetWorkspaceSessionHostIdForWorktree`) + writes pane bindings there. S4's reader ignores that partition, so supersession + is unaffected and the guarantee holds — but the write now lands somewhere + nothing reads. Harmless today, and a trap for whoever touches it next. Belongs + with step W. +- **The load fold moves leaf bindings only.** It deliberately does not touch + `tabsByWorktree[*].ptyId` in the ssh partition: that field is a tab-level + pointer no supersession path reads, and nulling it without a local counterpart + to move it to would be data loss. Narrowing the fold to what the defect + actually requires is why `reassignSshTargetId` needed no inversion. +- **The main-side detached branch still has no renderer signal.** S3 made the + main process stop lying; the affordance covers the renderer-initiated reattach + arms. A relay-driven reattach failure remains silent to the user. Closing it + needs a `pty:detached` channel — deferred deliberately, not overlooked. + +## Deliberately out of scope + +- The data-plane work (transport delivery guarantee, binary payloads, one credit + ledger). Real and separately justified, but independent of these goalposts. +- Rebuilding the authority architecture rejected earlier at +60,903 lines. + +## Owner decisions + +| | Decision | Status | +| --- | --- | --- | +| D1 | An unverifiable pane becomes visibly disconnected with two actions, instead of silently respawning | **Approved 2026-08-09** (implicit in approving the design; flagged for correction if not intended) | +| D2 | Whether the older gate/journey framing is rescoped or retired now that this design supersedes it | **Open** | diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md new file mode 100644 index 00000000000..0d20afdf08e --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/related-open-work.md @@ -0,0 +1,341 @@ +# Related GitHub work + +This is a curated overlap map, not a merge queue. States were verified against +GitHub on 2026-08-08 UTC and must be refreshed before any rebase, merge, close, +or implementation decision. + +Rules: + +- Do not stack PRs merely because they appear in the same section. +- “Conflict” means semantic or file overlap that requires one coherent design, + not an instruction to merge both. +- A related PR can supply diagnosis, tests, or a small reusable primitive + without becoming a dependency. +- Landing a related PR does not promote G0–G7 or one of the thirteen journeys + unless the final converged tree passes that gate's complete proof. +- GitHub issue/PR text is context, not authority over the user's goal. + +## Immediate incident and competing implementations + +- [Issue #11729 — SSH remote environment not restored after login on Rocky Linux + 9.4](https://github.com/stablyai/orca/issues/11729) — **open**. This is the + named STA-3077 customer incident. Its final acceptance must include Rocky + Linux SSH, repeated reconnect, the same PID/process/session/transcript + identity, restored workspace environment, visible pane count, and actual + remote relay PTY/process count. +- [PR #13111 — stop reconnect grafting panes, stacking leases, and respawning + live shells](https://github.com/stablyai/orca/pull/13111) — **open**, current + branch, green checks, no approval. It is narrow containment and leaves remote + shell reclamation unresolved. One major Docker settle review thread remains + [unresolved](https://github.com/stablyai/orca/pull/13111#discussion_r3739665033). +- [PR #12264 — stop reconnect from grafting ghost terminal + panes](https://github.com/stablyai/orca/pull/12264) — **open and conflicting**. + It targets the same incident and overlaps persistence, PTY, SSH relay, and E2E + files. Mine its diagnosis and discriminating artifacts; do not stack it + wholesale with #13111. +- [PR #12743 — contain reattach to existing panes, quarantine + orphans](https://github.com/stablyai/orca/pull/12743) — **open and + conflicting**. It shares `mayCreate: false` and broader relay coverage, but + quarantine must be judged against the no-reconciliation final design. +- [PR #12456 — reattach on `restoreRequired` instead of expiring a live + session](https://github.com/stablyai/orca/pull/12456) — **open**. It addresses + the #12448 branch through retry rather than cold respawn. Treat it as a + semantic alternative or complementary recovery path only after choosing one + exact restore protocol. +- [PR #9820 — reap orphaned relay PTYs before session-cap + exhaustion](https://github.com/stablyai/orca/pull/9820) — **open**. Its + time-based remote kill conflicts with unknown-is-not-dead and #13111's + deliberate non-kill behavior. Use it as incident evidence, not a safe default + dependency. +- [PR #12798 — opt-in zmx-backed durable SSH + terminals](https://github.com/stablyai/orca/pull/12798) — **open**. This is a + broad architectural alternative for final remote PTY ownership with direct + file overlap. Evaluate capability-by-capability; never blend it into the + current model accidentally. + +## Open incident issues that should become explicit oracles + +- [Issue #9819 — SSH relay leaks orphaned PTYs until the 50-session + cap](https://github.com/stablyai/orca/issues/9819) — explicit close, + disconnected close with a lost shutdown response, and worktree teardown must + durably retry retirement of the exact lease and process. Uncertain sessions + must survive visibly until resolved. Repeated reconnect must reclaim actual + relay PIDs/session slots rather than accumulate toward the cap. +- [Issue #9034 — SSH reconnect repeatedly spawns detached PTYs for one + pane](https://github.com/stablyai/orca/issues/9034) — count actual remote PTYs + and prove bounded one-pane ownership across repeated reconnect. +- [Issue #11006 — transient SSH disconnect treats live relay PTYs as dead and + duplicate-respawns agents](https://github.com/stablyai/orca/issues/11006) — + disconnect with a live process must remain unresolved/reattachable and must + not start another agent. +- [Issue #12447 — closed SSH tabs resurrect and auto-resume old + sessions](https://github.com/stablyai/orca/issues/12447) — explicit close must + persist its pending tombstone before a kill RPC, survive a lost reply plus app + and relay restart, retry after handshake, reclaim the exact remote PTY, and + never recreate UI or automatic resume. +- [Issue #12448 — `restoreRequired` is misreported as session + expiry](https://github.com/stablyai/orca/issues/12448) — source recovery must + replay/repaint on the same PID and binding without starting a replacement + `--resume`; identity drift, timeout, or unknown state must not become proof of + process death or cold resume authority. +- [Issue #12699 — SSH sleeping-agent wake sweep forks live remote + sessions](https://github.com/stablyai/orca/issues/12699) — all wake/resume + paths must share the same exact liveness and identity contract. +- [Issue #12683 — remote-runtime disconnect spends recovery on stale same-handle + reattach](https://github.com/stablyai/orca/issues/12683) — paired/web terminal + recovery must fence generations and retain automatic liveness. +- [Issue #10208 — duplicate terminal tab on worktree session + restore](https://github.com/stablyai/orca/issues/10208) — local/worktree + restore must prove the same pane and process rather than creating a duplicate. + +These issues are separate customer-visible symptoms. A single structural model +should make their shared invalid transitions impossible, while each issue keeps +its own discriminating regression journey. + +## Daemon, relay generation, and liveness work + +The acceptance matrix must also reproduce these open lifecycle incidents: + +- [Issue #11904 — tab close with a broken binding orphans the daemon + session](https://github.com/stablyai/orca/issues/11904) — close intent must be + durable before transport work and eventually retire the exact process. +- [Issue #8585 — detached relay generations are never + reaped](https://github.com/stablyai/orca/issues/8585) — termination must name + the exact relay generation; path/PID reuse or a failed connection cannot kill + a successor. +- [Issue #9138 — updates leave old daemon generations and sessions + alive](https://github.com/stablyai/orca/issues/9138) and + [issue #11342 — stale daemon/PTY generations leak across upgrade and + close](https://github.com/stablyai/orca/issues/11342) — upgrade must converge + without invisible sessions, guessed ownership, or unbounded retention. +- [Issue #10415 — Windows daemon crash on unkillable PTY and protocol + bump](https://github.com/stablyai/orca/issues/10415) — native Windows must + preserve live old-generation sessions or expose exact unresolved recovery; + one unkillable PTY cannot crash the daemon. +- [Issue #8275 — worktree teardown kills unrelated split-pane + sessions](https://github.com/stablyai/orca/issues/8275) — teardown and failure + must remain exact-namespace/exact-binding scoped. + +- [PR #12882 — publisher-owned daemon endpoint + replacement](https://github.com/stablyai/orca/pull/12882) — **open and + mergeable** when checked. It provides single-writer endpoint publication and + three-valued liveness. Reconcile it before editing overlapping daemon paths; + do not assume the older handoff's merge order is still current. +- [PR #11622 — fence daemon session ownership across + generations](https://github.com/stablyai/orca/pull/11622) — **open** and held + behind further audit per its description. Strong explicit + owned/unavailable/ambiguous semantics, but broad overlap makes it design input, + not an automatic dependency. +- [PR #9833 — report local PTY inventory + readiness](https://github.com/stablyai/orca/pull/9833) — **open**. Its + pending/ready/stale distinction is reusable context for unknown-is-not-dead. +- [PR #12760 — reap only the proven relay + generation](https://github.com/stablyai/orca/pull/12760) — **open**. It adds + exact relay generation ownership before termination and conflicts in + `ssh-relay-session.ts`; reconcile before designing remote shell cleanup. +- [PR #8618 — reap a detached relay after failed + reconnect](https://github.com/stablyai/orca/pull/8618) — **open** and an older + competing predecessor to #12760. Do not combine both ownership schemes. +- [PR #12749 — wait for a disconnected PTY owner's full + grace](https://github.com/stablyai/orca/pull/12749) — **open**. Useful owner + admission and grace context; timers may schedule retry but cannot independently + authorize takeover or destruction. +- [PR #12702 — stop sleeping-agent wake sweep from forking live SSH + sessions](https://github.com/stablyai/orca/pull/12702) — **open**. Complementary + #12699 work that must converge on the same ownership/liveness behavior; it + need not share an internal primitive unless the reconciled design calls for + one. +- [PR #10118 — durable persistence write + seam](https://github.com/stablyai/orca/pull/10118) — **open**. It overlaps + `persistence.ts` and is relevant to durability and main-thread performance; + reconcile before large persistence changes. + +## Paired runtime, remote server, and E2EE lifecycle work + +These open issues are mandatory inputs to the paired/remote/platform journeys: + +- [Issue #11495 — paired viewer loses attachments across update/restart while + host PTYs remain alive](https://github.com/stablyai/orca/issues/11495) — the + reported large host population is a strong restore/scale oracle; each pane + must return to the same host PTY/process. +- [Issue #11265 — paired terminals stall despite live TCP + sockets](https://github.com/stablyai/orca/issues/11265) — transport connectivity + is not delivery liveness; recovery must resume output without an unrelated RPC + kick. +- [Issue #11803 — remote stop respawns tabs from a stale + owner](https://github.com/stablyai/orca/issues/11803) — exact durable retirement + must remain retired after host/client restart. +- [Issue #12241 — paired-host session partitions grow without + bound](https://github.com/stablyai/orca/issues/12241) — paired and scale + journeys must prove bounded state and safe lost-consumer + retirement/compaction liveness. +- [Issue #11574 — same-server re-pairing changes host identity and strands + sessions](https://github.com/stablyai/orca/issues/11574) — stable final-host + identity must survive re-pairing without adopting another host. +- [Issue #9827 — investigate WSL session + restoration](https://github.com/stablyai/orca/issues/9827) and + [issue #11339 — WSL commands are not restored after + restart](https://github.com/stablyai/orca/issues/11339) — the daemon/WSL + journey must run on physical WSL and preserve both shell commands and agent + sessions. + +Additional open incident inputs must be triaged into a named journey or ruled +unrelated with explicit evidence and user acceptance: + +| Issue | Required question | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| [#11800 — paired null-PTY terminal ghosts](https://github.com/stablyai/orca/issues/11800) | Does host topology advancement retire exact placeholders without inference or a permanent timer reconciler? | +| [#9092 — remote runtime fails after macOS sleep](https://github.com/stablyai/orca/issues/9092) | Does generation-fenced reconnect recover after sleep with bounded resources and no identity change? | +| [#9585 — remote ghost tabs after host restart](https://github.com/stablyai/orca/issues/9585) | Are exit/retirement outcomes durable and idempotent across host restart? | +| [#9562 — remote host session restarts empty](https://github.com/stablyai/orca/issues/9562) | Does the complete boundary plus replay restore terminal state after server restart? | +| [#12568 — no PTY provider after SSH relay recovery/update](https://github.com/stablyai/orca/issues/12568) | Does lazy rediscovery restore the exact host/provider generation? | +| [#10385 — mobile remains falsely connected after RPC stalls](https://github.com/stablyai/orca/issues/10385) | Can E2EE/mobile liveness recover when transport looks connected but RPC is stalled? | +| [#12140 — mobile pairing socket closes immediately](https://github.com/stablyai/orca/issues/12140) | Do capability, identity, and E2EE negotiation fail explicitly and recover across LAN/Tailscale? | +| [#8129 — mobile events lost while disconnected](https://github.com/stablyai/orca/issues/8129) | Are semantic outcomes durable and replayed to a disconnected/backgrounded consumer? | + +- [PR #11575 — preserve host identity across same-server + re-pairing](https://github.com/stablyai/orca/pull/11575) — **open**. Direct + identity context for the paired reconnect journey. +- [PR #12768 — restore paired snapshots at the source + grid](https://github.com/stablyai/orca/pull/12768) — **open**. Complementary + host-snapshot authority work with renderer file overlap. +- [PR #9093 — recover remote runtime streams after client + wake](https://github.com/stablyai/orca/pull/9093) — **open**. Provides + generation-cancelled bounded-backoff reconnect context. +- [PR #10235 — self-heal remote subscriptions after server + restart](https://github.com/stablyai/orca/pull/10235) — **open**. Relevant to + E2EE session loss hidden by a live relay transport. +- [PR #11596 — recover falsely connected mobile RPC + sessions](https://github.com/stablyai/orca/pull/11596) — **open and + conflicting**. Broad mobile/E2EE reconnect context; do not absorb its parallel + lifecycle machinery without a full state-machine audit. +- [PR #12036 — stop blocking E2EE handshake on synchronous ACL + writes](https://github.com/stablyai/orca/pull/12036) — **open and conflicting**. + Preserve its metadata-versus-correctness durability distinction and measured + Windows performance constraint during identity lifecycle work. +- [PR #12134 — refresh stale capabilities before + pairing](https://github.com/stablyai/orca/pull/12134) — **open**. Capability + freshness and reauthentication are required mixed-version/pairing context. +- [PR #11804 — make remote stop retirement + durable](https://github.com/stablyai/orca/pull/11804) — **open**. Strong + complementary evidence for exact persisted owner routing and host-side + retirement acknowledgement. +- [PR #12987 — distinguish an absent worktree from an empty tab + list](https://github.com/stablyai/orca/pull/12987) — **open**. Reusable + unknown-versus-authoritative-empty semantics. +- [PR #12751 — preserve tabs when a snapshot reports + none](https://github.com/stablyai/orca/pull/12751) — **open** and a competing + less-exact alternative to #12987; its described multi-client non-convergence + must not become the final model. +- [PR #12339 — stop remote workspace pulls from resurrecting closed + tabs](https://github.com/stablyai/orca/pull/12339) — **open**. Direct topology + reconciliation context for explicit close and stale snapshots. +- [PR #13013 — stop stale snapshot applies from erasing fresh terminal + tabs](https://github.com/stablyai/orca/pull/13013) — **open**. Relevant legacy + snapshot race; decide whether its machinery is deleted by the final authority + cutover rather than layering another permanent reconciler. +- [PR #12903 — retry snapshot tabs whose paths resolve after + apply](https://github.com/stablyai/orca/pull/12903) — **open**. Deferred + hydration is relevant legacy-importer context and may become deletable after + authoritative namespace resolution; do not retain it by default. + +## Delivery, backpressure, and lifecycle alternatives + +- [Draft PR #12220 — keep SSH alive when a terminal outruns a slow + link](https://github.com/stablyai/orca/pull/12220) — **open draft**. Useful G3 + and performance evidence, but its own description has behavioral blockers and + a stalled-consumer wedge. Do not treat it as merge-ready. +- [PR #10744 — durable lost-worker archive and mixed-version SSH + revive](https://github.com/stablyai/orca/pull/10744) — **open** and broad. It + is major architectural context for exact close, outcomes, retirement, and + mixed versions, with substantial overlap. Mine proof and semantics rather + than stacking blindly. +- [PR #11801 — prune stale remote tab + ghosts](https://github.com/stablyai/orca/pull/11801) — **open**. Its snapshot + omission plus timer policy is relevant but may conflict with the rule that + absence/time alone cannot prove destructive cleanup. + +## Mandatory issue-to-journey matrix + +This table binds customer incidents to the thirteen journeys in +[`goalposts.md`](./goalposts.md#thirteen-required-production-journeys). Closing +an issue or merging a narrow PR does not remove its oracle. Each row must be red +on an unfixed baseline and green on the final candidate, or the user must accept +explicit evidence that the issue is unrelated. + +| Open issue(s) | Journey(s) | Mandatory discriminating outcome | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| [#11729](https://github.com/stablyai/orca/issues/11729) | 6, 9, 13 | Rocky Linux reconnect preserves the same workspace, transcript, pane, binding, PID, and environment without PTY growth. | +| [#9819](https://github.com/stablyai/orca/issues/9819), [#9034](https://github.com/stablyai/orca/issues/9034) | 6, 13 | Reconnect cannot accumulate detached PTYs/session slots; explicit close and teardown reclaim the exact process, including lost-response retry. | +| [#11006](https://github.com/stablyai/orca/issues/11006), [#12448](https://github.com/stablyai/orca/issues/12448), [#12699](https://github.com/stablyai/orca/issues/12699) | 2, 6 | Disconnect, source recovery, and wake keep the same live PID/session and never start a duplicate resume without proof of death. | +| [#12447](https://github.com/stablyai/orca/issues/12447), [#11904](https://github.com/stablyai/orca/issues/11904) | 2, 6 | Durable close intent survives lost RPC, app/daemon/relay restart, retries exact retirement, reclaims the process, and never resurrects UI. | +| [#10208](https://github.com/stablyai/orca/issues/10208) | 1, 9 | Worktree switch/restore preserves one exact pane/process and does not duplicate a tab. | +| [#8585](https://github.com/stablyai/orca/issues/8585) | 2, 6, 12 | Cleanup affects only the proven relay generation across reconnect and version change; ID/path reuse cannot kill a successor. | +| [#9138](https://github.com/stablyai/orca/issues/9138), [#11342](https://github.com/stablyai/orca/issues/11342) | 2, 12, 13 | Upgrade/restart converges old daemon generations and their PTYs without invisible sessions, leaks, or destructive guessing. | +| [#8275](https://github.com/stablyai/orca/issues/8275) | 1, 2, 5 | Worktree teardown cannot terminate a sibling namespace or split-pane session. | +| [#10415](https://github.com/stablyai/orca/issues/10415) | 1, 2, 5, 12 | Native Windows handles an unkillable PTY and protocol bump without daemon crash, live-session loss, or cross-session damage. | +| [#9827](https://github.com/stablyai/orca/issues/9827), [#11339](https://github.com/stablyai/orca/issues/11339) | 2, 9 | Physical WSL restart restores the same shell command and agent session under the exact namespace/binding. | +| [#12568](https://github.com/stablyai/orca/issues/12568) | 3, 6, 12 | Relay recovery/update lazily rediscovers the same SSH host/provider and restores only its exact sessions. | +| [#11495](https://github.com/stablyai/orca/issues/11495), [#11574](https://github.com/stablyai/orca/issues/11574) | 4, 8, 10, 12, 13 | Paired restart/re-pair preserves stable host identity and every reported live host PTY at scale across independent updates. | +| [#11265](https://github.com/stablyai/orca/issues/11265), [#9092](https://github.com/stablyai/orca/issues/9092) | 8, 13 | Live-but-stalled transport and sleep/wake automatically resume bounded delivery without an unrelated RPC kick or identity replacement. | +| [#11803](https://github.com/stablyai/orca/issues/11803), [#9585](https://github.com/stablyai/orca/issues/9585), [#11800](https://github.com/stablyai/orca/issues/11800) | 3, 8 | Remote stop/host topology outcome remains durably retired after restart; no stale-owner tab or null-PTY ghost returns. | +| [#9562](https://github.com/stablyai/orca/issues/9562), [#8129](https://github.com/stablyai/orca/issues/8129) | 3, 8, 13 | Complete boundary plus replay restores disconnected/restarted consumers without an empty session or lost semantic outcome. | +| [#12241](https://github.com/stablyai/orca/issues/12241) | 8, 11, 13 | Paired partitions and retained-consumer state remain bounded; lost-consumer retirement safely restores compaction liveness. | +| [#12683](https://github.com/stablyai/orca/issues/12683) | 8, 12 | Stale same-handle reconnect is generation-fenced and automatic recovery remains live across independent peer versions. | +| [#10385](https://github.com/stablyai/orca/issues/10385), [#12140](https://github.com/stablyai/orca/issues/12140) | 8, 10, 12 | False-connected or rejected E2EE/pairing state fails explicitly and reauthenticates without stale identity, capability, or session adoption. | + +Journeys 4, 5, 7, 10, and 11 remain mandatory even where no single issue fully +specifies them. This matrix adds incident oracles; it never narrows the journey +definitions. + +## Original containment/refactor line + +- [PR #12600 — break Activity and terminal React 185 + loops](https://github.com/stablyai/orca/pull/12600) — **merged**. It is current + containment/background context, not the whole program and not a release gate. +- [PR #12634 — replace React 185 timers with structural + handoffs](https://github.com/stablyai/orca/pull/12634) — **open**. This is the + direct structural follow-up for deleting #12600's window/timer brakes and must + be reconciled with the broader terminal-session model. +- [PR #13110 — sandboxed preload artifact + guard](https://github.com/stablyai/orca/pull/13110) — **open**, independent of + #13111, green at its head, no approval. Its unresolved major review says raw + source matching can reject safe strings/comments; resolve or supersede that + [finding](https://github.com/stablyai/orca/pull/13110#discussion_r3739646949) + before calling it ready. + +## Already merged base context + +These were described as open in older notes but were merged when this handoff +was written: + +- [PR #12474 — isolate same-path folder workspace PTY + identity](https://github.com/stablyai/orca/pull/12474) +- [PR #12477 — stop a closed remote workspace window reading + Ready](https://github.com/stablyai/orca/pull/12477) +- [PR #12600 — React/terminal loop containment](https://github.com/stablyai/orca/pull/12600) + +Refresh `main` and recalculate conflicts; do not follow the stale sequence +`#12882 → #12474 → #12477 → #13111` as written. + +## Required GitHub refresh before action + +At minimum, re-query: + +```bash +gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +gh pr view 13110 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +gh issue view 11729 --repo stablyai/orca --json state,title,updatedAt,url +``` + +Refresh the complete catalog in one batched request: + +```bash +handoff_query_text="$(tr '\n' ' ' < docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql)" +gh api graphql -f query="$handoff_query_text" +``` + +The query includes current PR state/check summaries and unresolved-thread state +for #13110/#13111. Read PR bodies/diffs only for the slice being changed. Do not +close, comment, merge, push, or checkout a PR from the handoff audit without +explicit authorization. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html b/docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html new file mode 100644 index 00000000000..4497d1f42e4 --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/report.html @@ -0,0 +1,376 @@ + + + + + + + Terminal Session Correctness — Design & Test Report + + + +
+
+

Terminal Session Correctness

+

Design & test report · branch nwparker/sta-3077-reattach-pane-cardinality · PRs #13110, #13111

+
+ +
+
+214
Production lines
+
+60,903
Rejected approach
+
12
Oracles that bite
+
2 / 13
Journeys proven
+
0 / 8
Gates proven
+
2
Claims retracted
+
+ +
+

The one-sentence version

+

Every bug here is the same bug — identity compared with the wrong key, or not compared at all. A previous attempt added a second identity system beside the first at a cost of 60,903 lines and fixed none of the three root causes. This fixes all three in 214.

+
+ +

1 · What sits behind one terminal

+

Five separate things line up behind the rectangle you see.

+
+
Anatomy of a terminal⤢ zoom
+

The real computer

Orca's bookkeeping

What you see

bound to

points at

claims

runs on

Pane
a rectangle in a tab

Binding
this pane uses that shell

Lease
we own a remote shell

PTY
the shell process

Host
your Mac, or an SSH box

The real computer

Orca's bookkeeping

What you see

bound to

points at

claims

runs on

Pane
a rectangle in a tab

Binding
this pane uses that shell

Lease
we own a remote shell

PTY
the shell process

Host
your Mac, or an SSH box

+
+

The PTY is the real shell doing the work. The binding is the note connecting it to your pane. On a remote machine a lease is a second note saying “we still own that shell.” Every failure below is one of those notes pointing at the wrong thing.

+ +

2 · The reported failure

+
+
Reconnect growth — a real customer report⤢ zoom
+

2 terminals

network hiccup

reconnect

19 terminals

hiccup

20 terminals
most are ghosts

2 terminals

network hiccup

reconnect

19 terminals

hiccup

20 terminals
most are ghosts

+
+

The remote machine also fills with unused shells until it refuses to start more. A second, worse variant: an AI coding agent resumed twice — two claude processes writing to one conversation file. One report reached five.

+ +

3 · Three root causes

+ +

Cause 1 — the lease forgot its pane

+

A lease identified a shell but never checked which pane it belonged to. Every reconnect added a note; nothing removed one.

+
+
Lease accumulation⤢ zoom
+

reconnect

reconnect

Pane A

lease → shell-1

lease → shell-2

lease → shell-3

nothing retires
the old notes

next reconnect restores
all three

reconnect

reconnect

Pane A

lease → shell-1

lease → shell-2

lease → shell-3

nothing retires
the old notes

next reconnect restores
all three

+
+

FIXED A lease is identified by its pane. One pane, one live lease. Superseded leases are marked expired, not killed — the remote shell keeps running, because losing a note is not proof the shell died.

+ +

Cause 2 — reconnecting could create panes

+
+
Reattach decision⤢ zoom
+

yes

no · before

no · after

reconnect finds a lease

does its pane
still exist?

reattach — correct

invent the pane
👻 ghost appears

refuse
shell left running, unbound

yes

no · before

no · after

reconnect finds a lease

does its pane
still exist?

reattach — correct

invent the pane
👻 ghost appears

refuse
shell left running, unbound

+
+

FIXED Reattach binds only. Opening a new terminal still creates, exactly as before.

+ +

Cause 3 — a living shell reported as dead

+ + + + + + +
Real meaningReported asConsequence
This shell is goneSESSION_EXPIREDcorrect
Shell is fine — output pipe needs re-establishingSESSION_EXPIREDrespawn → agent resumed twice
+

FIXED The second case has its own message, and respawn now requires positive proof. Anything unrecognised is “unknown”, which never justifies a replacement.

+ +

4 · The rule underneath all of it

+
+
Unknown is not dead⤢ zoom
+

shell exited

can't reach it

timed out

absent from a list

3 failed retries

evidence

what does it prove?

DEAD
cleanup allowed

UNKNOWN

nothing destructive
wait · retry · ask

shell exited

can't reach it

timed out

absent from a list

3 failed retries

evidence

what does it prove?

DEAD
cleanup allowed

UNKNOWN

nothing destructive
wait · retry · ask

+
+

The liveness check also became three-valued. It previously returned yes/no, so a provider whose socket was down could only answer “no” — meaning dead — when the honest answer was “I can’t see”.

+ +

5 · What changed

+ + + + + + + + + + +
#ChangePrevents
1Reattach binds, never createsGhost panes
2Leases identified by pane2 → 19 → 20 growth
3Heal already-corrupted lease dataExisting installs stuck at 20
4“Needs re-establishing” ≠ “expired”Duplicate agent resume
5Liveness can answer “unknown”Live shells declared dead
6One PTY’s failure can’t drop the shared channelOne bad pane killing every session on a host
+
+
Change 6 — blast radius⤢ zoom
+

after

pane 3 can't re-prove
its output stream

park pane 3 only

its shell keeps running
siblings untouched

before

pane 3 can't re-prove
its output stream

give up after N tries

drop the whole
host connection

every pane, file transfer
and git command dies

after

pane 3 can't re-prove
its output stream

park pane 3 only

its shell keeps running
siblings untouched

before

pane 3 can't re-prove
its output stream

give up after N tries

drop the whole
host connection

every pane, file transfer
and git command dies

+
+
+

No wire change

+

The keystroke guard adds nothing to the payload. Orca already keeps two lookup tables in lock-step, so when they disagree that is proof the id is stale — a purely server-side check.

+
+ +

6 · How we decide a test is worth trusting

+
+

This happened to us twice

+

An end-to-end test was reported as proving the reconnect fix. It passed with the fix and with it removed — retracted. Separately, a guard shipped with no production caller at all; every test stayed green because they called the capability directly.

+
+
+
The four-step proof⤢ zoom
+

stays green

1 · run it
green

2 · delete a guard
in production code

3 · run it
MUST go red

4 · restore
green again

worthless
say so — don't ship
it as proof

stays green

1 · run it
green

2 · delete a guard
in production code

3 · run it
MUST go red

4 · restore
green again

worthless
say so — don't ship
it as proof

+
+

Delete a real guard, not the test. Prefer one clause at a time — a mutation reddening exactly one claim proves the claims independently. Watch it fail yourself: six of twelve oracles were personally re-verified, and two of those re-checks overturned the original claim.

+ +

7 · Why “same id” is not good enough

+

The most valuable lesson here. Break the quit path so it destroys shells instead of detaching, then restart:

+ 
tab id    same ✓
+pane id   same ✓
+pty id    same ✓
+OS pid    13756 → 8852   ✗
+start     02:01:29.337 → 02:01:34.728   ✗
+
+Every id-only restart test stays green here.
+Your shell is gone.
+
+
Proving the process really survived⤢ zoom
+
KernelShellOrcaTestKernelShellOrcaTestrestart the apptype "echo MARKER=$$"through the real write pathMARKER=13756start time of 13756?02:01:29.337ask againMARKER=13756start time of 13756?02:01:29.337 — same process ✓
KernelShellOrcaTestKernelShellOrcaTestrestart the apptype "echo MARKER=$$"through the real write pathMARKER=13756start time of 13756?02:01:29.337ask againMARKER=13756start time of 13756?02:01:29.337 — same process ✓
+
+ +

8 · Journey status

+ + + + + + + + + + + + +
#JourneyStateWhy
1Local restart — macOS · Linux · WindowsPROVENClause-selective on all three platforms
2Daemon & physical WSLPROVENClause-selective on macOS, Linux, WSL2
4Concurrent multi-hostBY CONSTRUCTIONA mux is per-host — dispose cannot cross hosts, so there is nothing to break
3Lazy discoveryNO SELECTIVE MUTATIONRemoving the real guard breaks setup before the clause is reached
6Docker MaxSessions=1PARTIAL2 clauses redden; a third survived four guard removals
12Mixed versionsWITHDRAWNThe token never crosses a version boundary in production
13Performance & scaleIN PROGRESS1 of 10 dimensions measured; rest running
5, 10, 11Device proof · identity reset · namespace admissionNO SUBJECTThe code does not exist. No oracle is writable at any effort.
+
+
Why a property may resist proof⤢ zoom
+

yes

no — true by
construction

no code exists

property to prove

is a shipped guard
protecting it?

remove it → red
provable

nothing to remove
not provable by mutation

nothing to test
not writable

yes

no — true by
construction

no code exists

property to prove

is a shipped guard
protecting it?

remove it → red
provable

nothing to remove
not provable by mutation

nothing to test
not writable

+
+
+

Read this carefully

+

“True by construction” is a good outcome — the property holds because of how the code is shaped, not because a guard watches it. It simply cannot be demonstrated by breaking something. Reporting it as proven would be false.

+
+ +

9 · Traps that can fake a result

+ + + + + + + + +
TrapEffectMitigation
Shared temp directory — the harness writes a pointer to a machine-wide pathConcurrent runs can fake a red and hide a real oneIsolated TMPDIR per run
Stale build — e2e runs a built appYou tested the old codeCheck build timestamp against source edit
Mutation silently didn’t applyTest stays green, looks like “no teeth” — wrong conclusionVerify the mutation landed before believing the result
Serial mode reports “did not run”A skip mistaken for a passRe-run the others alone under the same mutation
+

Two of our own tests were broken in ways only real hardware revealed: one failed on every Linux run (a single unpolled read where its sibling polls for 15s), and one could not execute at all on Windows (echo $$ and ps don’t exist there).

+ +

10 · Open decisions

+ + + + + + + + +
IDDecisionBlocks
D2Re-scope G4? It demands an architecture where 0 of 394 files exist, measured at +62k LOC — the approach that already failed at 0/13.G4, G1, G3
D3Strike or defer J5 / J10 / J11? They have no production subject.3 journeys
D4Architectural gates need a “reviewed diff + file list” receipt; the doc’s red-SHA template makes them structurally unpromotable.G0, G1, G4
D5The input quarantine is load-bearing, not superseded. Deleting it re-opens a verified rm -rf hazard. Amend the clause, or accept something weaker on remote hosts?G6
+
+

The hazard, verified

+

Disable the quarantine and the oracle reproduces it exactly: echo hi; rm -rf x arrives as cho hi; rm -rf x. The shell rejects cho and then runs rm -rf x. The replacement was built and costed at roughly +140 lines to delete 88 — and it does not work on remote hosts at all, because old hosts never publish the incarnation it needs.

+
+ +

11 · Corrections made along the way

+

Included because a report without them is not trustworthy.

+ + + + + + + + + + +
ClaimOutcome
“The reconnect e2e proves the fix”Retracted — passed with the fix removed, twice, isolated
“Cross-version tests confirm old clients are safe”Withdrawn — they exercise a route production never takes
“mayCreate fixes reattach”Was inert — no production caller for several commits
“A branded binding type is the fix”Rejected — three claims false; would have added a second identity system
“Fixtures compile as production”Corrected — none ship; the sweep was stopped
“bash 5.3.9 causes the WSL failure”Corrected — macOS runs the same version and passes
+ +
+ Generated for review alongside design-overview.md, test-overview.md and goalposts.md. + Click any diagram to zoom · scroll to scale · drag to pan. +
+
+ +
+ × +
+
scroll to zoom · drag to pan · Esc to close
+
+ + + + diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md new file mode 100644 index 00000000000..62c4274d5ea --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/resume-plan.md @@ -0,0 +1,275 @@ +# Resume plan and validation record + +This plan preserves the comprehensive objective while allowing independently +safe stacks. It does not require reviving the 814-path construction snapshot. + +## First safe commands + +Run from the primary worktree before editing: + +```bash +pwd +git status --short --branch --untracked-files=all +git rev-parse HEAD origin/main +git rev-list --left-right --count HEAD...origin/main +git branch --list 'nwparker/react185-authority-snapshot' +git show --stat --oneline fddb19f6977 +gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +``` + +Expected handoff checkpoint before other changes: + +- branch `nwparker/sta-3077-reattach-pane-cardinality`; +- HEAD `5369479be2953f45cea9ab5cfcde756dd9660548`; +- snapshot branch and commit present; +- only this handoff folder and its `.gitignore` allow-list are new handoff work. + +If the branch, HEAD, or dirty paths differ, inventory and preserve them before +continuing. Never reset, clean, overwrite, or switch away from unknown user work. + +## Phase 0 — Reconcile reality before design or code + +1. Refresh `origin/main` with `git fetch origin main`—never `git pull`—and + refresh the live state/reviews of every PR in + [`related-open-work.md`](./related-open-work.md) that touches the intended + slice. +2. Recalculate the candidate merge base, categorized LOC, changed-file overlap, + and failing/obsolete tests. +3. Freeze the program accounting baseline before implementation. The default is + `5ed45739e94bdf6460364e033bfcec9b32c0b42a` (the base of #12600). Build an + aggregate carry-in ledger for every program-attributable PR already merged + after it; do not measure only the last PR in a stack. +4. Read the final diff of #13111, #12264, and #12743 with `gh pr view` and + `gh pr diff`, or in disposable isolated worktrees. Never use `gh pr checkout` + or switch branches in the primary worktree. Select or design one coherent + lease/reattach model; do not combine mutually inconsistent quarantine, kill, + and non-kill policies. +5. Reconcile #12882 before overlapping daemon-endpoint integration and #12760 + before overlapping relay-generation cleanup. Neither blocks unrelated SSH + proof work by declaration. +6. Treat the preserved snapshot as a local-only mine, not a branch to finish. + Inspect it with `git show fddb19f6977:` or in a disposable isolated + worktree; never switch the primary worktree to it. + +Decision output required before implementation: + +- exact incident mechanisms being eliminated; +- exact authority/identity semantics retained or changed; +- explicit remote-shell lifecycle and user recovery behavior; +- explicit lost-consumer liveness/compaction behavior; +- PR/stack boundaries and dependency order; +- production modules expected to be deleted; and +- tests that will discriminate the baseline from the candidate; and +- a predeclared performance protocol with fixed workloads, samples, + statistical/noise bounds, deterministic counters, and raw-data locations. + +## Phase 1 — Build discriminating journeys first + +Before relying on a code change, create or repair oracles that reproduce the +failure on an unfixed baseline. + +### STA-3077 Docker SSH oracle + +The Docker journey must: + +- run a real OpenSSH server with `MaxSessions=1` verified from the server; +- create a known pane and record tab, leaf, binding, relay PTY, and remote PID; +- seed or naturally reproduce duplicate/stale lease state from the field; +- exercise disconnect, client restart, reconnect, and authority/relay restart; +- wait an explicit settle interval after successful reattach; +- recensus visible panes, tabs, exact bindings, persisted leases, relay PTYs, + and actual remote processes/PIDs; +- prove no pane or shell is created, rebound, lost, or silently killed; +- prove explicit unresolved recovery for state that cannot be established; and +- exercise explicit close, lost shutdown response, reconnect retry, and + worktree teardown; prove exact remote PID/session-slot reclamation while an + uncertain sibling survives; and +- fail for the intended reason on the unfixed baseline. + +A clean reconnect that passes on both revisions is not this oracle. + +### Duplicate-agent oracle family + +Cover transient SSH loss, `restoreRequired`, sleeping-agent wake, worktree +switch, and closed-tab restart. Assert process count and transcript/session +ownership, not just pane/store state. No path may start a second `--resume` +process while the first process is live or unknown. + +### Daemon and remote oracles + +Cover unknown inventory, endpoint handover, daemon restart, relay generation +reuse, client sleep/wake, remote-runtime restart, and stale subscription repair. +A retry limit may end an attempt or surface a failure; it may not itself prove a +session dead or authorize destructive replacement. + +## Phase 2 — Reconcile and approve the long-term design + +The implementation design must explicitly settle: + +- final-host identity versus client routing aliases; +- canonical host-local namespace for worktree, folder, floating, drive, and UNC + forms; +- structural one-active-binding/lease ownership rather than optional metadata + plus repair scans; +- exact operation fencing across every transport; +- explicit detach, close, retirement, orphan recovery, and process-reclamation + operations; +- durable outcome/replay semantics across app, renderer, host, and transport + restarts; +- mixed-version capability and isolated legacy behavior; +- lost-device consumer retirement and compaction liveness; +- one-way migration/cutover; and +- the four-role deletion oracle and minimised, justified production-code target. + +Reuse existing pane, PTY, incarnation, CAS, and three-valued liveness primitives +where their semantics match. Do not equate client routing IDs with host identity +or namespace without proving the full semantics. + +If the chosen design differs from the preserved normative design, write an +explicit mapping for every invariant and every journey, identify what changes, +show equal or stronger safety/liveness, and obtain the user's approval. A +handoff author or implementation agent cannot approve that scope change alone. + +## Phase 3 — Implement as a convergent stack + +Stacks are allowed for reviewability. Temporary production stopgaps are not the +end state. + +A sensible dependency shape is: + +1. discriminating journey harnesses and behavior oracles; +2. final shared identity, liveness, and exact-binding primitives; +3. local/daemon/WSL/SSH/relay/paired/remote adapters using those primitives; +4. durable outcome, projection, reset, and recovery lifecycle; +5. one-way migration and capability cutover; +6. deletion of superseded writers, ledgers, repair scans, timers used as verdicts, + and duplicate state machines; retain `terminal-input-quarantine.ts` until a + replacement passes its destructive-input oracle; and +7. full platform, skew, performance, packaging, and independent review proof. + +Each stack layer must be safe and tested, but the comprehensive status remains +0/8 until the final integrated candidate satisfies the complete gates. Do not +advertise the authoritative capability or claim program completion mid-stack. + +## Phase 4 — Run the complete proof matrix + +Use [`goalposts.md`](./goalposts.md) as the checklist. All thirteen production +journeys and every row in the mandatory +[`issue-to-journey matrix`](./related-open-work.md#mandatory-issue-to-journey-matrix) +are required. At minimum the final matrix must include: + +- real macOS, Linux floor, Windows, and WSL; +- local, daemon, direct SSH, nested SSH, paired runtime, and remote server; +- one Docker host with `MaxSessions=1` and two simultaneous Docker hosts; +- git worktree, folder, floating, drive, and UNC workspaces; +- old/new peers in both directions at every changed app/daemon, SSH relay, + paired-runtime, remote-server, and mobile/E2EE boundary; +- app, renderer, daemon, relay, and remote-runtime restart; +- crash cuts around durable operations and identity reset; +- unknown/timeout/disconnect/lost-response/concurrent-replacement cases; +- actual remote PID and process accounting; and +- A/B performance and memory results under the predeclared protocol, including + raw samples, confidence intervals, deterministic counters, and leak slopes. + +## Phase 5 — Delete, census, rebase, and review + +1. Trace every new production module from a real entrypoint. +2. Delete unreachable modules and fixtures in production trees. +3. Delete all superseded legacy writers and reconciliation state machines after + cutover. +4. Rebase onto the actual PR base. +5. Rerun all journeys and static/package checks on the rebased SHA. +6. Report production, tests, docs, CI/runner, generated, and vendored LOC + separately. +7. Minimise aggregate production source net LOC against the frozen pre-program + baseline, including all program-attributable carry-in and stack layers, and + justify every net addition as required by D1. +8. Run an independent repository review and the release-readiness checklist. +9. Resolve every P0–P2 finding. +10. Only then prepare the comprehensive PR and ask the user to approve any + deliberate design change. + +## Validation status at this handoff + +This was a documentation and read-only audit turn. No product test command was +rerun, so this handoff claims no fresh local product-test pass. + +Exact validation commands run for this handoff: + +```bash +git status --short --branch --untracked-files=all +git rev-parse HEAD origin/main +git rev-list --left-right --count HEAD...origin/main +git merge-base HEAD origin/main +git diff --numstat --no-renames "$(git merge-base HEAD origin/main)"..HEAD +gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url +handoff_query_text="$(tr '\n' ' ' < docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql)" +gh api graphql -f query="$handoff_query_text" +pnpm exec oxfmt --check docs/reference/terminal-session-correctness-handoff-2026-08-07/*.md docs/reference/terminal-session-correctness-handoff-2026-08-07/*.graphql +git diff --check +``` + +Outcomes: + +- repository commands exited 0 and produced the branch/SHA/distance recorded in + `README.md`; +- the current-HEAD numstat reproduced `+309` production, `+914` tests/E2E, + `+146` docs, `+43` CI/config, and `+1,412` total net LOC; +- the reproducible GraphQL catalog returned all 65 referenced items and one + unresolved, non-outdated review thread on each of #13110 and #13111; +- all 30 linked issues appear in the mandatory issue-to-journey matrix and the + reusable GraphQL query; +- #13111 was open, non-draft, mergeable/clean, with 47 successful and 5 skipped + checks and no approval decision; and +- formatting and whitespace checks exited 0 after documentation formatting. + +Verified in this worktree: + +- repository branch, HEAD, merge base, local `origin/main`, branch distance, + and clean pre-documentation status; +- snapshot branch/commit presence and the 814-path preservation claim; +- current source implementation of the narrow #13111 behavior and gaps; +- current categorized #13111 diff census; +- original G0–G7, thirteen journeys, and PR proof contract; and +- live GitHub PR/issue state through batched GraphQL queries. + +Live GitHub verification at the time of writing showed: + +- #13111: open, non-draft, mergeable/clean, checks successful, no approval, one + unresolved major review thread; +- #13110: open, non-draft, checks successful, no approval, one unresolved major + review thread; +- #12264 and #12743: open competing STA-3077 implementations; +- #12474, #12477, and #12600: merged; and +- the other items in `related-open-work.md`: open unless explicitly labeled + merged or draft. + +These states are ephemeral and must be refreshed. + +Historical pause receipts from the 60,903-line construction snapshot remain +construction evidence only. They do not prove the current branch, G0–G7, or any +required journey. + +## Stop conditions + +Stop and return to the user before: + +- narrowing or replacing a named goalpost; +- accepting a correctness/performance tradeoff; +- deciding that remote orphan shells may be killed or leaked permanently; +- changing wire semantics without a compatibility design; +- shipping a positive-production-LOC comprehensive stack; +- merging, closing, commenting on, or force-updating external PRs; or +- deleting preserved user work. + +## Short takeover prompt + +> Continue the comprehensive terminal-session correctness program from +> [`README.md`](./README.md). Preserve the fixed status of **0/8 proven gates +> and 0/13 proven journeys** until every proof in +> [`goalposts.md`](./goalposts.md) passes on one rebased candidate. Reconcile the +> conflicting open work in [`related-open-work.md`](./related-open-work.md), +> build discriminating real SSH/daemon/paired/remote/cross-platform journeys, +> remove root causes and superseded reconciliation, minimise and justify every +> production-code addition per D1, and finish with no correctness or performance +> regression. diff --git a/docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md b/docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md new file mode 100644 index 00000000000..29083dfbb2a --- /dev/null +++ b/docs/reference/terminal-session-correctness-handoff-2026-08-07/test-overview.md @@ -0,0 +1,277 @@ +# Terminal session correctness — test design + +How we decide a test is worth trusting, what we built, and what we could not +prove. Read [`design-overview.md`](./design-overview.md) first. + +--- + +## 1. The problem with "the tests pass" + +A green test proves nothing on its own. It might pass because the code is +correct, or because the test never looked. + +```mermaid +flowchart TD + T["Test is green"] --> Q{"Would it go red if
we broke the code?"} + Q -->|"Yes"| G["Real evidence"] + Q -->|"No"| W["Worthless.
Green either way."] + Q -->|"Never checked"| U["Unknown.
Treat as worthless."] + + style G fill:#e6ffe6,stroke:#00aa00 + style W fill:#ffcccc,stroke:#cc0000 + style U fill:#fff4e6,stroke:#dd8800 +``` + +This is not hypothetical. **It happened to us twice:** + +- An end-to-end test was reported as proving the reconnect fix. It passed with + the fix **and** with the fix removed. Retracted. +- A guard was added, every test stayed green, and the guard had **no production + caller at all**. The capability existed; nothing used it. Tests passed the + whole time because they called the capability directly. + +--- + +## 2. The rule we adopted: the four-step proof + +Nothing counts as evidence until it has been through this loop. + +```mermaid +flowchart LR + S1["1. Run it
green"] --> S2["2. Delete the guard
from production code"] + S2 --> S3["3. Run it
MUST go red"] + S3 --> S4["4. Put the guard back
green again"] + S3 -.->|"stays green"| F["Test is worthless.
Say so. Don't ship it
as proof."] + + style S3 fill:#e6ffe6,stroke:#00aa00 + style F fill:#ffcccc,stroke:#cc0000 +``` + +Three rules that make this honest rather than theatre: + +**Delete a real guard, not the test.** The mutation goes in _production_ code. If +you tweak the test until it fails, you have proved nothing. + +**Prefer one clause at a time.** If a test has three claims, a good mutation +reddens exactly one and leaves the other two green. That proves the claims +_independently_ rather than as a bundle. + +**Watch it fail yourself.** A report saying "it went red" is not the same as +seeing it. Of the twelve oracles here, six were personally re-verified, and two +of those re-checks overturned the original claim. + +### The trap that nearly fooled us twice + +```mermaid +flowchart TD + M["Apply mutation"] --> C{"Did the mutation
actually land?"} + C -->|"No — silent no-op"| X["Test stays green.
You conclude 'no teeth'.
Wrong conclusion."] + C -->|"Yes"| R["Now the result
means something"] + + style X fill:#ffcccc,stroke:#cc0000 +``` + +Twice a search-and-replace matched the wrong line and changed nothing. The test +stayed green and looked like a failed oracle. **Always verify the mutation landed +before believing the result.** + +--- + +## 3. The three layers of test + +```mermaid +flowchart TD + subgraph L1["Unit — milliseconds"] + U1["Pure logic.
Does the rule compute
the right answer?"] + end + subgraph L2["Integration — seconds"] + I1["Real Electron app.
Real restarts.
Real shell processes."] + end + subgraph L3["End-to-end — minutes"] + E1["Real Docker container
running real OpenSSH.
Real network faults."] + end + L1 --> L2 --> L3 + L1 -.->|"fast, narrow"| N1[" "] + L3 -.->|"slow, real"| N2[" "] + style N1 fill:none,stroke:none + style N2 fill:none,stroke:none +``` + +The rule of thumb: **assert the strongest thing you can actually observe.** + +--- + +## 4. Why "same id" is not good enough + +The most valuable lesson from this work. + +A restart test naturally asks: _is it the same terminal?_ The easy check is +whether the ids match. That check is **too weak**, and we can prove it. + +```mermaid +flowchart TD + B["Break the quit path so it
destroys shells instead of
detaching from them"] --> R["Restart the app"] + R --> C1["tab id: same ✅"] + R --> C2["pane id: same ✅"] + R --> C3["pty id: same ✅"] + R --> C4["OS process: DIFFERENT ❌
pid 13756 → 8852"] + C3 --> W["A weak test passes here.
Your shell is gone."] + C4 --> S["Our test fails here."] + + style W fill:#ffcccc,stroke:#cc0000 + style S fill:#e6ffe6,stroke:#00aa00 +``` + +So the test asks the shell to report its **own process id** through the real +input path, then reads that process's **kernel start time**. Same id _and_ same +start time means the same process really survived — a recycled pid cannot pass. + +```mermaid +sequenceDiagram + participant T as Test + participant A as Orca + participant S as Shell + participant K as Kernel + T->>A: type "echo MARKER=$$" + A->>S: (through the real write path) + S-->>T: MARKER=13756 + T->>K: start time of 13756? + K-->>T: 02:01:29.337 + Note over T: restart the app + T->>S: ask again + S-->>T: MARKER=13756 + T->>K: start time of 13756? + K-->>T: 02:01:29.337 ✅ same process +``` + +--- + +## 5. What is covered + +### Journey 1 — a terminal survives restart · **PROVEN** + +Two claims: the same pane, binding and **OS process** survive a reload and an app +restart; and a stale operation is refused. + +| Platform | Result | Mutation A
(destructive quit) | Mutation B
(remove stale-write guard) | +| ------------------ | ------ | --------------------------------- | ----------------------------------------- | +| macOS | 2 pass | red | red | +| Linux, native | 2 pass | red | red | +| Windows 11, native | 2 pass | **red — only claim 1** | **red — only claim 2** | + +The two mutations hit **different claims**, which is what proves them separately. + +### Journey 2 — daemon and WSL · **PROVEN** + +| Environment | Result | Mutation A
(liveness can't say "unknown") | Mutation B
(widen owner fallback) | +| --------------- | ------ | --------------------------------------------- | ------------------------------------- | +| macOS | 3 pass | red — claim 1 only | red — claim 3 only | +| Linux, native | 3 pass | red — claim 1 only | red — claim 3 only | +| WSL2 on Windows | 3 pass | red — claim 1 only | — | + +> **A subtlety worth knowing.** These tests run in order, so when the first one +> fails the others report "did not run" — that is a _skip_, not a pass. To prove +> the mutation really only hit claim 1, we re-ran claims 2 and 3 **alone, with +> the mutation still applied**, and watched them stay green. + +### Other oracles that bite + +| Oracle | Mutation that reddens it | +| ---------------------------- | ------------------------------------------------ | +| Reconnect grafts no pane | Remove "bind, never create" | +| Superseded-keystroke fence | Remove the guard from the handlers | +| Guard is actually wired up | Remove the call sites, keep the function | +| Workspace ids don't collide | Restore the suffix-stripping comparison | +| Wire compatibility | Restore the wrong "expired" message | +| `rm -rf` hazard stays closed | Disable the quarantine → `cho hi; rm -rf x` runs | + +That last one is why we did **not** delete the quarantine module. + +--- + +## 6. What we could not prove, and why + +This is the honest half. + +```mermaid +flowchart TD + P["Property we want
to prove"] --> Q{"Is there a shipped
guard protecting it?"} + Q -->|"Yes"| M["Remove it → test reddens.
Provable"] + Q -->|"No — it's true
by construction"| C["Nothing to remove.
Not provable by mutation"] + Q -->|"No code exists
at all"| N["Nothing to test.
Not writable"] + + style M fill:#e6ffe6,stroke:#00aa00 + style C fill:#fff4e6,stroke:#dd8800 + style N fill:#ffcccc,stroke:#cc0000 +``` + +| Journey | Why not proven | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| Two hosts don't interfere | True **by construction** — each host has its own channel, so there is no shared thing to break. Four attempts to redden it failed, correctly. | +| Lazy host discovery | One claim has no mutation that hits it alone; removing the real guard breaks setup before the claim is reached. | +| `MaxSessions=1` reconnect | Two claims redden. A third survived **four** different guard removals — nothing shipped protects it. | +| Mixed versions, live | The new message **never crosses a version boundary** in production. Our in-process tests exercise a route the real app doesn't take. Withdrawn. | +| Performance | 1 of 10 required dimensions measured. The keystroke guard costs ~14ns, but measured outside the real app. | +| Three others | The features they test **do not exist** in the code. No test can be written. | + +**"True by construction" is a good outcome, not a failure** — it means the +property holds because of how the code is shaped, not because a guard is +watching. It just can't be demonstrated by breaking something. + +--- + +## 7. Environment traps that can fake a result + +Real problems we hit, worth knowing before trusting any run. + +```mermaid +flowchart TD + subgraph T1["Shared temp directory"] + A1["Test harness writes a pointer
to a machine-wide temp path"] --> A2["Two runs at once
overwrite each other"] + A2 --> A3["Can fake a red
and hide a real one"] + end + subgraph T2["Stale build"] + B1["e2e runs a built app"] --> B2["Edit source, forget rebuild"] + B2 --> B3["You tested the old code"] + end + style A3 fill:#ffcccc,stroke:#cc0000 + style B3 fill:#ffcccc,stroke:#cc0000 +``` + +So every run isolates its temp directory, and we check build timestamps against +source edits before believing a result. + +Two more, found by running on machines we don't develop on: + +- A test failed on **every** Linux run. Not a product bug: it read a value once, + immediately after a reload, where its sibling helper polls for 15 seconds. +- A test could not run on **Windows at all** — it used `echo $$` and `ps`, which + don't exist there. The Windows claim wasn't unproven, it was _unprovable_. + +Both were bugs in tests we wrote, found only by running on the real platform. + +--- + +## 8. Scoreboard + +```mermaid +pie showData + title Journeys + "Proven with discrimination" : 2 + "Oracle exists, cannot discriminate" : 5 + "No code exists to test" : 3 + "Not yet attempted" : 3 +``` + +| Measure | Value | +| ------------------------------- | ----------- | +| Journeys proven | **2 of 13** | +| Gates proven | **0 of 8** | +| Oracles that bite | 12 | +| …personally re-verified | 6 | +| Claims retracted after checking | 2 | +| Production code added | +214 lines | + +The retraction count is the number to look at. Two claims that looked green were +withdrawn after checking. That is the process working — and the reason to trust +the two journeys that survived it. diff --git a/package.json b/package.json index 1c2c1deb2df..2f2954020df 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,9 @@ "test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs", "test:e2e:ssh-docker-watcher-isolation": "node config/scripts/run-ssh-docker-watcher-isolation-e2e.mjs", "test:e2e:ssh-docker-terminal-parking": "node config/scripts/run-ssh-docker-terminal-parking-e2e.mjs", + "test:e2e:ssh-docker-reconnect-pane-cardinality": "node config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs", + "test:e2e:ssh-docker-maxsessions-identity": "node config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs", + "test:e2e:ssh-docker-two-hosts-isolation": "node config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs", "test:e2e:nested-runtime-ssh": "node config/scripts/run-nested-runtime-ssh-e2e.mjs", "test:e2e:source-control-scale": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/source-control-large-file-count.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "win-update-e2e": "node tests/tools/win-update-e2e/run.mjs", @@ -124,6 +127,7 @@ "bench:cold-park-resource": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs", "bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs", "test:e2e:remote-bulk-open-freeze": "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "test:e2e:daemon-session-liveness": "pnpm run ensure:electron-runtime && pnpm exec playwright test --config tests/playwright.daemon-liveness.local.config.ts", "test:e2e:ssh-docker-bulk-open-freeze": "node config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs", "repro:live-remote-bulk-open-freeze": "node config/scripts/live-remote-bulk-open-freeze-repro.mjs", "repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs" diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index 1e41e9de9ee..1fe5bc9770e 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -1911,6 +1911,20 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(adapter.hasPty(id)).toBe(true) }) + // Why: off-socket the cache can only be missing exits it never received, so a + // miss is ignorance. Answering false there authorizes a respawn over a live shell. + it('answers unknown, not absent, for an uncached id while disconnected', async () => { + const { id } = await adapter.spawn({ cols: 80, rows: 24 }) + const unknownId = 'repo::/repo/unknown@@deadbeef' + expect(adapter.hasPty(unknownId)).toBe(false) + + await server.shutdown() + await waitFor(() => !(adapter as unknown as { client: DaemonClient }).client.isConnected()) + + expect(adapter.hasPty(unknownId)).toBe(null) + expect(adapter.hasPty(id)).toBe(true) + }) + it('returns active sessions', async () => { await adapter.spawn({ cols: 80, diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index b2ccf55f6d7..49bc8b5a34b 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -940,8 +940,10 @@ export class DaemonPtyAdapter implements IPtyProvider { return providerSequence ? { providerSequence } : undefined } - hasPty(id: string): boolean { - return this.activeSessionIds.has(id) + hasPty(id: string): boolean | null { + // Why null off-socket: the cache only tracks exits it received, so a miss while + // disconnected (or before the first listSessions) is ignorance, not absence. + return this.activeSessionIds.has(id) ? true : this.client.isConnected() ? false : null } async probePtyLiveness(id: string): Promise { diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index 41cc8016169..7ddc240cc74 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -529,6 +529,33 @@ describe('DaemonPtyRouter', () => { expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session') }) + // Why: an unreachable generation cannot certify that another generation's session + // is gone — collapsing its null to false hands a respawn a fabricated death. + it('answers unknown when any adapter cannot see the session', () => { + const current = createAdapter('current') + const legacy = createAdapter('legacy') + legacy.hasPty = vi.fn(() => null) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + expect(router.hasPty('unmapped-session')).toBe(null) + }) + + it('does not report terminal_gone for inspection when an adapter cannot answer', async () => { + const current = createAdapter('current') + const legacy = createAdapter('legacy') + legacy.hasPty = vi.fn(() => null) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + + await expect(router.inspectProcess('unmapped-session')).resolves.toEqual({ + foregroundProcess: null, + hasChildProcesses: false + }) + + // Unproven routes must not be memoized as an owner. + const unanimous = new DaemonPtyRouter({ current: createAdapter('c2'), legacy: [] }) + await expect(unanimous.inspectProcess('unmapped-session')).rejects.toThrow('terminal_gone') + }) + it('discovers an unmapped live session from one coalesced inventory', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy', ['surviving-session']) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 5104268cbef..a3962e511be 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -77,12 +77,15 @@ export class DaemonPtyRouter implements IPtyProvider { return await this.adapterFor(id).attach(id) } - hasPty(id: string): boolean { + hasPty(id: string): boolean | null { const routed = this.sessionAdapters.get(id) if (routed) { return routed.hasPty(id) } - return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id)) + // Why: one adapter's ignorance is not another's absence proof — only a unanimous + // proven-absence across every generation may read as false. + const answers = this.allAdapters().map((adapter) => adapter.hasPty(id)) + return answers.includes(true) ? true : answers.includes(null) ? null : false } async probePtyLiveness(id: string): Promise { @@ -314,14 +317,19 @@ export class DaemonPtyRouter implements IPtyProvider { } private adapterForInspection(sessionId: string): DaemonPtyAdapter { - const adapter = + const owner = this.sessionAdapters.get(sessionId) ?? - this.allAdapters().find((candidate) => candidate.hasPty(sessionId)) - if (!adapter) { + this.allAdapters().find((candidate) => candidate.hasPty(sessionId) === true) + if (owner) { + this.sessionAdapters.set(sessionId, owner) + return owner + } + // Why: only unanimous proven absence is gone; an adapter that cannot answer routes + // to current without memoizing a route we never established. + if (this.hasPty(sessionId) === false) { throw new Error('terminal_gone') } - this.sessionAdapters.set(sessionId, adapter) - return adapter + return this.current } private allAdapters(): DaemonPtyAdapter[] { diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index 6b473537e9f..1982c4add1b 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -471,6 +471,22 @@ describe('DegradedDaemonPtyProvider', () => { }) }) + // Why: while degraded, a provider that cannot answer must not let inspection + // manufacture terminal_gone — that verdict retires a pane that may still be live. + it('answers unknown, and refuses terminal_gone, when no provider can answer', async () => { + const current = createDaemonAdapter('daemon') + const fallback = createProvider('fallback') + current.hasPty = vi.fn(() => null) + fallback.hasPty = vi.fn(() => null) + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + expect(provider.hasPty('unmapped-session')).toBe(null) + await expect(provider.inspectProcess('unmapped-session')).resolves.toEqual({ + foregroundProcess: null, + hasChildProcesses: false + }) + }) + it('caches a provider discovered by hasPty before routing later operations', () => { const current = createDaemonAdapter('daemon', ['daemon-session']) const fallback = createProvider('fallback') @@ -662,3 +678,22 @@ describe('DegradedDaemonPtyProvider', () => { expect(fallback.listProcesses).toHaveBeenCalledTimes(3) }) }) + +// A memoized route outlives the session it was established for: listProcesses +// drops ids missing from an authoritative inventory without an exit fanout. So a +// mapped owner that cannot answer must stay unknown — coercing it to a liveness +// proof is worse than the absence it replaced, because callers skip the real probe. +it('keeps a mapped owner that cannot answer unknown, and still probes', async () => { + const current = createDaemonAdapter('current', ['s1']) + const fallback = createProvider('fallback') + const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback }) + + expect(provider.hasPty('s1')).toBe(true) + + current.hasPty = vi.fn(() => null) + expect(provider.hasPty('s1')).toBeNull() + + current.probePtyLiveness = vi.fn(async () => null) + await provider.probePtyLiveness('s1') + expect(current.probePtyLiveness).toHaveBeenCalled() +}) diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index 545f66ca892..e5b67478812 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -89,14 +89,25 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { attach = (id: string): ReturnType => attachDaemonOwnedSession(this.providerFor(id), this.fallback, id) - hasPty(id: string): boolean { + hasPty(id: string): boolean | null { const mapped = this.sessionProviders.get(id) - return mapped ? (mapped.hasPty?.(id) ?? true) : this.findProviderForExistingSession(id) !== null + if (mapped) { + // Why not `?? true`: a route outlives the session it was memoized for, so an + // owner that cannot answer must stay unknown rather than become a liveness proof. + return mapped.hasPty ? mapped.hasPty(id) : true + } + if (this.findProviderForExistingSession(id)) { + return true + } + // Why: one provider that cannot answer makes absence unknown; only unanimous proof is false. + return this.allProviders().every((provider) => provider.hasPty?.(id) === false) ? false : null } async probePtyLiveness(id: string): Promise { const mapped = this.sessionProviders.get(id) - if (mapped && (mapped.hasPty?.(id) ?? true)) { + // Why only an explicit true short-circuits: an owner that cannot answer must + // fall through to the real probe instead of skipping it on a fabricated proof. + if (mapped && (mapped.hasPty ? mapped.hasPty(id) === true : true)) { return true } return await this.ownerRecovery.probe(id) @@ -180,9 +191,9 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { return this.providerFor(id).getForegroundProcess(id) } inspectProcess(id: string) { - return this.hasPty(id) - ? inspectPtyProviderProcess(this.providerFor(id), id) - : Promise.reject(new Error('terminal_gone')) + return this.hasPty(id) === false + ? Promise.reject(new Error('terminal_gone')) + : inspectPtyProviderProcess(this.providerFor(id), id) } async confirmForegroundProcess(id: string): Promise { return this.providerFor(id).confirmForegroundProcess?.(id) ?? null diff --git a/src/main/ipc/ssh-pty-output-intake-test-harness.ts b/src/main/ipc/__tests__/ssh-pty-output-intake-test-harness.ts similarity index 97% rename from src/main/ipc/ssh-pty-output-intake-test-harness.ts rename to src/main/ipc/__tests__/ssh-pty-output-intake-test-harness.ts index 9209b897284..c3fe0b26ac9 100644 --- a/src/main/ipc/ssh-pty-output-intake-test-harness.ts +++ b/src/main/ipc/__tests__/ssh-pty-output-intake-test-harness.ts @@ -3,7 +3,7 @@ import { SshPtyOutputIntake, type SshPtyOutputDataEvent, type SshPtyOutputIntakeDependencies -} from './ssh-pty-output-intake' +} from '../ssh-pty-output-intake' export function sshPtyOutputEvent( overrides: Partial = {} diff --git a/src/main/ipc/pty-superseded-operation-fence.test.ts b/src/main/ipc/pty-superseded-operation-fence.test.ts new file mode 100644 index 00000000000..f357954bc26 --- /dev/null +++ b/src/main/ipc/pty-superseded-operation-fence.test.ts @@ -0,0 +1,118 @@ +/** + * A mutating operation must not reach a PTY that has been superseded for its + * pane. Main keeps `ptyPaneKey` and `paneKeyPtyId` in lock-step, so their + * disagreement is proof the caller's id is stale — not an inference from + * absence, which is why an id with no recorded pane stays permitted. + * + * The fence lives in the main process because the renderer queues input: a + * keystroke buffered before a reattach would otherwise land on the successor. + */ +import { describe, expect, it } from 'vitest' + +/** Mirrors the maps and predicate in pty.ts. Kept structural so the oracle + * tests the RULE, not one implementation of it. */ +function makeFence() { + const ptyPaneKey = new Map() + const paneKeyPtyId = new Map() + const bind = (paneKey: string, ptyId: string): void => { + ptyPaneKey.set(ptyId, paneKey) + paneKeyPtyId.set(paneKey, ptyId) + } + const isSuperseded = (ptyId: string): boolean => { + const paneKey = ptyPaneKey.get(ptyId) + if (paneKey === undefined) { + return false + } + const currentPtyId = paneKeyPtyId.get(paneKey) + return currentPtyId !== undefined && currentPtyId !== ptyId + } + return { bind, isSuperseded, ptyPaneKey, paneKeyPtyId } +} + +const PANE = 'tab-1:3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + +describe('superseded PTY operation fence', () => { + it('refuses an operation whose pane has since bound a different PTY', () => { + const fence = makeFence() + fence.bind(PANE, 'pty-old') + // The pane reattaches onto a fresh PTY; the renderer still holds pty-old. + fence.bind(PANE, 'pty-new') + + expect(fence.isSuperseded('pty-old')).toBe(true) + expect(fence.isSuperseded('pty-new')).toBe(false) + }) + + it('permits the current PTY for its own pane', () => { + const fence = makeFence() + fence.bind(PANE, 'pty-live') + + expect(fence.isSuperseded('pty-live')).toBe(false) + }) + + // Unknown is not stale. Orphan cleanup targets exactly these ids, so refusing + // them would break the operation that reclaims leaked shells. + it('permits a PTY that owns no pane', () => { + const fence = makeFence() + + expect(fence.isSuperseded('pty-orphan')).toBe(false) + }) + + it('permits a PTY whose pane record was dropped', () => { + const fence = makeFence() + fence.bind(PANE, 'pty-live') + fence.paneKeyPtyId.delete(PANE) + + expect(fence.isSuperseded('pty-live')).toBe(false) + }) + + it('keeps sibling panes independent', () => { + const fence = makeFence() + const other = 'tab-1:8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + fence.bind(PANE, 'pty-a') + fence.bind(other, 'pty-b') + fence.bind(PANE, 'pty-a2') + + expect(fence.isSuperseded('pty-a')).toBe(true) + expect(fence.isSuperseded('pty-b')).toBe(false) + }) +}) + +describe('the fence is wired into every mutating handler', () => { + it('combines supersession and incarnation in the write fence', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + const start = source.indexOf('const isCurrentPtyWrite') + + expect(start).toBeGreaterThan(0) + expect(source.slice(start, start + 300)).toContain('isSupersededPtyId') + expect(source.slice(start, start + 300)).toContain('ptyIncarnationById') + }) + + // The capability existed for months and was never called from these handlers. + // Pin the call site, not the capability — that is the failure this program hit. + it.each(['pty:write', 'pty:writeAccepted', 'pty:resize', 'pty:signal'])( + '%s consults the fence', + async (channel) => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + // Anchor on the registration, not the first mention of the name. + const start = source.search( + new RegExp(`ipcMain\\.(?:on|handle)\\(\\s*'${channel.replace(':', ':')}'`) + ) + expect(start, `${channel} handler not found`).toBeGreaterThan(0) + expect(source.slice(start, start + 700)).toContain( + channel.startsWith('pty:write') ? 'isCurrentPtyWrite' : 'isSupersededPtyId' + ) + } + ) + + // pty:kill is deliberately NOT fenced: a superseded PTY is orphaned, and + // reclaiming it is exactly what the orphan-cleanup callers ask for. + it('leaves pty:kill unfenced on purpose', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + const start = source.search(/ipcMain\.handle\(\s*'pty:kill'/) + expect(start).toBeGreaterThan(0) + expect(source.slice(start, start + 500)).not.toContain('isSupersededPtyId') + }) +}) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index d44ad8ef651..29d2feb10b9 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -245,6 +245,7 @@ import { getLocalPtyProvider, isCurrentPtyExit, restorePtyIncarnation, + bindPaneShell, type PrepareCodexSessionResume } from './pty' import { __resetPersistedWindowsPathCacheForTests } from '../pty/windows-environment-path' @@ -5289,7 +5290,10 @@ describe('registerPtyHandlers', () => { paneKey?: string tabId?: string }) => ({ - id: 'ssh-pty' + id: 'ssh-pty', + // The host attests the shell's identity at spawn; the lease has to carry it or the + // reconnect fence has nothing to compare and attaches by pty id alone. + incarnationId: 'inc-host-ssh-pty' }) ) const store = { @@ -5375,18 +5379,23 @@ describe('registerPtyHandlers', () => { worktreeId: 'wt-1', tabId: 'tab-1', leafId, + // Without this the field exists, loads and is read — and is never written, so the + // reconnect fence silently permits everything while every other clause stays green. + incarnationId: 'inc-host-ssh-pty', state: 'attached' }) ) - expect(store.persistPtyBinding).toHaveBeenCalledWith( - { - worktreeId: 'wt-1', - tabId: 'tab-1', - leafId, - ptyId: 'ssh-pty' - }, - 'ssh:ssh-1' - ) + // STA-3077 step P inverted this: SSH pane bindings no longer go to a + // per-target `ssh:` partition. One home (`local`) only, so the + // hostId argument is gone — pin its absence, not its old value. + expect(store.persistPtyBinding).toHaveBeenCalledWith({ + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId, + ptyId: 'ssh-pty', + incarnationId: 'inc-host-ssh-pty' + }) + expect(store.persistPtyBinding.mock.calls.at(-1)).toHaveLength(1) store.upsertSshRemotePtyLease.mockClear() store.persistPtyBinding.mockClear() @@ -6820,6 +6829,38 @@ describe('registerPtyHandlers', () => { expect(fallbackShutdown).not.toHaveBeenCalled() }) + // Why: before the swap lands the pre-swap local provider owns no daemon id, so its + // "not in my table" would answer an authoritative dead for every restored session. + it('waits for the desktop startup barrier before pty:hasPty answers for a daemon id', async () => { + const barrier = makeDeferred() + const awaitLocalPtyProviderStartup = vi.fn(() => barrier.promise) + registerPtyHandlers( + mainWindow as never, + undefined, + undefined, + undefined, + undefined, + undefined, + { + awaitLocalPtyProviderStartup + } + ) + + const pending = handlers.get('pty:hasPty')!(null, { id: 'daemon-session' }) as Promise< + boolean | null + > + await Promise.resolve() + expect(awaitLocalPtyProviderStartup).toHaveBeenCalledTimes(1) + + installObservableDaemonTestProvider() + ;(getLocalPtyProvider() as { hasPty?: (id: string) => boolean | null }).hasPty = vi.fn( + (id: string) => id === 'daemon-session' + ) + barrier.resolve() + + await expect(pending).resolves.toBe(true) + }) + it('waits for the desktop startup barrier before runtime local kills resolve the provider', async () => { const barrier = makeDeferred() const awaitLocalPtyProviderStartup = vi.fn(() => barrier.promise) @@ -7840,6 +7881,7 @@ describe('registerPtyHandlers', () => { id: opts.sessionId ?? 'daemon-pty' })), write, + hasPty: vi.fn(() => true), resize: vi.fn(args.resize ?? (() => {})), getAppliedSize: vi.fn(args.getAppliedSize ?? (async () => args.applied)), kill: vi.fn(), @@ -8064,6 +8106,107 @@ describe('registerPtyHandlers', () => { expect(write).not.toHaveBeenCalled() }) + it('does not forward queued host input after the pane rebinds', async () => { + const write = setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 } }) + let resolveClaim: (claimed: boolean) => void = () => {} + const claimRemoteDesktopHost = vi.fn( + () => + new Promise((resolve) => { + resolveClaim = resolve + }) + ) + const runtime = { + setPtyController: vi.fn(), + createPreAllocatedTerminalHandle: vi.fn(() => null), + registerPty: vi.fn(), + getDriver: vi.fn(() => ({ kind: 'idle' })), + claimRemoteDesktopHost, + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const spawn = await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24, env: {} }) + const id = (spawn as { id: string }).id + const leafId = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + const bindingStore = { + persistPtyBinding: vi.fn(() => true), + getWorkspaceSession: vi.fn(() => getDefaultWorkspaceSession()) + } + bindPaneShell({ + store: bindingStore, + worktreeId: 'repo-1:wt-1', + tabId: 'tab-1', + leafId, + ptyId: id + }) + const claim = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:claimViewport') + const writeEvent = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:write') + + claim?.[1](mainWindowIpcEvent, { id, cols: 125, rows: 48 }) + writeEvent?.[1](mainWindowIpcEvent, { id, data: 'queued' }) + const accepted = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id, + data: 'accepted' + }) + bindPaneShell({ + store: bindingStore, + worktreeId: 'repo-1:wt-1', + tabId: 'tab-1', + leafId, + ptyId: 'pty-successor' + }) + resolveClaim(true) + + await expect(accepted).resolves.toBe(false) + await Promise.resolve() + expect(write).not.toHaveBeenCalled() + clearProviderPtyState(id) + clearProviderPtyState('pty-successor') + }) + + it('does not forward queued host input after the PTY incarnation changes', async () => { + const write = setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 } }) + let resolveClaim: (claimed: boolean) => void = () => {} + const runtime = { + setPtyController: vi.fn(), + createPreAllocatedTerminalHandle: vi.fn(() => null), + registerPty: vi.fn(), + getDriver: vi.fn(() => ({ kind: 'idle' })), + claimRemoteDesktopHost: vi.fn( + () => + new Promise((resolve) => { + resolveClaim = resolve + }) + ), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const spawn = await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24, env: {} }) + const id = (spawn as { id: string }).id + restorePtyIncarnation(id, 'incarnation-old') + const claim = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:claimViewport') + const writeEvent = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:write') + + claim?.[1](mainWindowIpcEvent, { id, cols: 125, rows: 48 }) + writeEvent?.[1](mainWindowIpcEvent, { id, data: 'queued' }) + const accepted = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id, + data: 'accepted' + }) + restorePtyIncarnation(id, 'incarnation-new') + resolveClaim(true) + + await expect(accepted).resolves.toBe(false) + await Promise.resolve() + expect(write).not.toHaveBeenCalled() + clearProviderPtyState(id) + }) + it('does not populate the remote reclaim cache when only a phone drives', async () => { const resizeSpy = vi.fn() setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 }, resize: resizeSpy }) @@ -8665,12 +8808,17 @@ describe('registerPtyHandlers', () => { } }): Promise<{ id: string }> } + let emitExit: ((event: { exitCode: number; signal: number }) => void) | undefined + const kill = vi.fn(() => emitExit?.({ exitCode: 0, signal: 0 })) const proc = { onData: vi.fn(), - onExit: vi.fn(), + onExit: vi.fn((callback) => { + emitExit = callback + return makeDisposable() + }), write: vi.fn(), resize: vi.fn(), - kill: vi.fn(), + kill, process: 'zsh', pid: 12345 } @@ -8725,7 +8873,7 @@ describe('registerPtyHandlers', () => { expect(store.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ expectedSourceBinding }) ) - expect(proc.kill).toHaveBeenCalledOnce() + expect(kill).toHaveBeenCalledOnce() }) it('reports lower-owner commit before rejecting an early-exited runtime incarnation', async () => { @@ -9639,6 +9787,9 @@ describe('registerPtyHandlers', () => { ) expect(runtime.noteTerminalSpawnCommand).not.toHaveBeenCalled() expect(store.persistPtyBinding).toHaveBeenCalledOnce() + // STA-3077 step P inverted this: the reattach writer no longer passes a + // hostId slot at all (it used to pass `undefined` here for the local case), + // so the call is now strictly single-argument. expect(store.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ worktreeId, @@ -9650,9 +9801,10 @@ describe('registerPtyHandlers', () => { ptyId: 'pty-persisted-owner', incarnationId: 'inc-stale-owner' } - }), - undefined + }) ) + expect(store.persistPtyBinding.mock.calls[0]!).toHaveLength(1) + expect(getPtyIdForPaneKey(paneKey)).toBe('pty-persisted-owner') expect( mainWindow.webContents.send.mock.calls.filter(([channel]) => channel === 'pty:spawned') ).toHaveLength(1) @@ -10351,9 +10503,139 @@ describe('registerPtyHandlers', () => { expect(runtime.onPtyExit).toHaveBeenCalledWith('pty-already-retired-owner', 0, undefined) }) - it('retires a dead owner from the exact SSH host session before fresh recovery', async () => { + // Sibling of the clause above, and the reason that one is driven by a proof. Retiring an owner + // authorizes a replacement carrying the pane's agent resume payload, so a bare not-found must + // not do it: a replaced relay answers exactly that for shells its predecessor still runs. + it('does not retire an SSH owner, nor respawn it, on a bare not-found', async () => { + const connectionId = 'ssh-unproven-stable-pane' + const tabId = 'tab-unproven-ssh-owner' + const leafId = '35353535-3535-4535-8535-353535353535' + const paneKey = makePaneKey(tabId, leafId) + const worktreeId = 'repo-ssh::/remote/unproven-stable-pane' + const livePtyId = `ssh:${connectionId}@@live-relay-pty` + const freshPtyId = `ssh:${connectionId}@@fresh-relay-pty` + const remoteSpawn = vi.fn(async (options: { attachOnly?: boolean; command?: string }) => { + if (options.attachOnly) { + // The reattach mints this only after verifying the relay's observed exit names this shell. + // A bare not-found reaches here too and must NOT retire the owner — pinned by the sibling + // clause below, because retirement authorizes a replacement carrying the resume payload. + throw new Error('PTY "live-relay-pty" not found') + } + return { id: freshPtyId, incarnationId: 'inc-fresh-ssh-owner' } + }) + registerSshPtyProvider(connectionId, { + spawn: remoteSpawn, + write: vi.fn(), + resize: vi.fn(), + shutdown: vi.fn(), + sendSignal: vi.fn(), + getCwd: vi.fn(), + getInitialCwd: vi.fn(), + clearBuffer: vi.fn(), + acknowledgeDataEvent: vi.fn(), + onData: vi.fn(() => () => {}), + onReplay: vi.fn(() => () => {}), + onExit: vi.fn(() => () => {}), + listProcesses: vi.fn(), + hasChildProcesses: vi.fn(), + getForegroundProcess: vi.fn(), + serialize: vi.fn(), + revive: vi.fn(), + getDefaultShell: vi.fn(), + getProfiles: vi.fn() + } as never) + let session = { + tabsByWorktree: { + [worktreeId]: [{ id: tabId, worktreeId, ptyId: livePtyId }] + }, + terminalLayoutsByTabId: { + [tabId]: { + root: { type: 'leaf' as const, leafId }, + activeLeafId: leafId, + expandedLeafId: null, + ptyIdsByLeafId: { [leafId]: livePtyId } + } + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: 'inc-live-ssh-owner' } + } + const store = { + // STA-3077 step P inverted these: the reader/retirer used to target the + // per-target `ssh:` partition while the renderer published + // pane membership to `local` — two homes, so supersession no-opped. Both + // sides now address the single default (local) partition, i.e. no hostId. + getWorkspaceSession: vi.fn((requestedHostId?: string) => { + expect(requestedHostId).toBeUndefined() + return session + }), + setWorkspaceSession: vi.fn((next, requestedHostId?: string) => { + expect(requestedHostId).toBeUndefined() + session = next + }), + flushOrThrow: vi.fn(), + persistPtyBinding: vi.fn(), + upsertSshRemotePtyLease: vi.fn(), + removeSshRemotePtyLease: vi.fn(), + markSshRemotePtyLease: vi.fn() + } + const runtime = { + setPtyController: vi.fn(), + resolveTerminalPane: vi.fn(() => { + throw new Error('terminal_not_found') + }), + createPreAllocatedTerminalHandle: vi.fn(() => 'term-fresh-ssh-owner'), + registerPreAllocatedHandleForPty: vi.fn(), + beginPtyRegistration: vi.fn(), + cancelPendingPtyRegistration: vi.fn(), + assertPtyRegistrationAllowed: vi.fn(), + registerPty: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + seedHeadlessTerminal: vi.fn(), + getDriver: vi.fn(() => ({ kind: 'host' })), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + + try { + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + await expect( + handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + cwd: '/remote/unproven-stable-pane', + command: 'codex resume a-session-that-may-still-be-running', + connectionId, + worktreeId, + tabId, + leafId, + env: { + ORCA_PANE_KEY: paneKey, + ORCA_TAB_ID: tabId, + ORCA_WORKTREE_ID: worktreeId + } + }) + ).rejects.toThrow(/not found/i) + + // Only the attach ran: no replacement carrying the resume payload. + expect(remoteSpawn).toHaveBeenCalledTimes(1) + expect(remoteSpawn.mock.calls[0]?.[0]).toMatchObject({ attachOnly: true }) + // The pane keeps its owner, so the shell stays reattachable. + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + } finally { + unregisterSshPtyProvider(connectionId) + } + }) + + it('retires a dead owner from the local session — the one pane-binding home — before fresh recovery', async () => { const connectionId = 'ssh-dead-stable-pane' - const hostId = `ssh:${connectionId}` const tabId = 'tab-dead-ssh-owner' const leafId = '34343434-3434-4434-8434-343434343434' const paneKey = makePaneKey(tabId, leafId) @@ -10362,7 +10644,8 @@ describe('registerPtyHandlers', () => { const freshPtyId = `ssh:${connectionId}@@fresh-relay-pty` const remoteSpawn = vi.fn(async (options: { attachOnly?: boolean; command?: string }) => { if (options.attachOnly) { - throw new Error('PTY "dead-relay-pty" not found') + // Proven exit: only this error authorizes retiring the owner (bare not-found must not). + throw new Error('SSH_SESSION_EXPIRED: dead-relay-pty') } return { id: freshPtyId, incarnationId: 'inc-fresh-ssh-owner' } }) @@ -10402,12 +10685,16 @@ describe('registerPtyHandlers', () => { terminalPtyIncarnationsByPaneKey: { [paneKey]: 'inc-dead-ssh-owner' } } const store = { + // STA-3077 step P inverted these: the reader/retirer used to target the + // per-target `ssh:` partition while the renderer published + // pane membership to `local` — two homes, so supersession no-opped. Both + // sides now address the single default (local) partition, i.e. no hostId. getWorkspaceSession: vi.fn((requestedHostId?: string) => { - expect(requestedHostId).toBe(hostId) + expect(requestedHostId).toBeUndefined() return session }), setWorkspaceSession: vi.fn((next, requestedHostId?: string) => { - expect(requestedHostId).toBe(hostId) + expect(requestedHostId).toBeUndefined() session = next }), flushOrThrow: vi.fn(), @@ -10470,16 +10757,25 @@ describe('registerPtyHandlers', () => { expect(remoteSpawn.mock.calls[1]?.[0]).toMatchObject({ command: 'codex resume exact-dead-ssh-provider-session' }) - expect(store.setWorkspaceSession).toHaveBeenCalledWith(expect.anything(), hostId) + // STA-3077 step P inverted these: the retirement write and the fresh + // binding write both land in the single default (local) partition now, so + // no hostId is passed. The dead owner must still actually be retired. + expect(store.setWorkspaceSession).toHaveBeenCalledOnce() + expect(store.setWorkspaceSession.mock.calls[0]!).toHaveLength(1) + expect( + store.setWorkspaceSession.mock.calls[0]![0].terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[ + leafId + ] + ).not.toBe(deadPtyId) expect(store.persistPtyBinding).toHaveBeenCalledWith( expect.objectContaining({ worktreeId, tabId, leafId, ptyId: freshPtyId - }), - hostId + }) ) + expect(store.persistPtyBinding.mock.calls.at(-1)!).toHaveLength(1) } finally { unregisterSshPtyProvider(connectionId) } @@ -10825,15 +11121,15 @@ describe('registerPtyHandlers', () => { state: 'attached' }) ) - expect(store.persistPtyBinding).toHaveBeenCalledWith( - { - worktreeId: 'wt-remote', - tabId: 'tab-remote', - leafId, - ptyId: 'ssh:ssh-1@@relay-pty' - }, - 'ssh:ssh-1' - ) + // STA-3077 step P inverted this: the headless binding writer no longer + // targets `ssh:`; one home (`local`) only, so no hostId argument. + expect(store.persistPtyBinding).toHaveBeenCalledWith({ + worktreeId: 'wt-remote', + tabId: 'tab-remote', + leafId, + ptyId: 'ssh:ssh-1@@relay-pty' + }) + expect(store.persistPtyBinding.mock.calls[0]!).toHaveLength(1) expect(store.persistPtyBinding.mock.invocationCallOrder[0]!).toBeLessThan( store.upsertSshRemotePtyLease.mock.invocationCallOrder[0]! ) @@ -10977,15 +11273,15 @@ describe('registerPtyHandlers', () => { persistHostSessionBinding: true }) - expect(store.persistPtyBinding).toHaveBeenCalledWith( - { - worktreeId: 'wt-remote', - tabId: 'tab-remote', - leafId, - ptyId: 'ssh:ssh-reattach-ok@@relay-pty' - }, - 'ssh:ssh-reattach-ok' - ) + // STA-3077 step P inverted this: the reattach binding refresh writes to the + // single default (local) partition, so the `ssh:` hostId is gone. + expect(store.persistPtyBinding).toHaveBeenCalledWith({ + worktreeId: 'wt-remote', + tabId: 'tab-remote', + leafId, + ptyId: 'ssh:ssh-reattach-ok@@relay-pty' + }) + expect(store.persistPtyBinding.mock.calls.at(-1)!).toHaveLength(1) expect(store.upsertSshRemotePtyLease).toHaveBeenCalledWith( expect.objectContaining({ targetId: 'ssh-reattach-ok', @@ -17660,6 +17956,31 @@ describe('registerPtyHandlers', () => { expect(mockProc.proc.write).toHaveBeenNthCalledWith(2, 'tail') }) + it('stops a chunked write when the PTY incarnation changes between chunks', async () => { + vi.useFakeTimers() + const mockProc = createMockProc() + spawnMock.mockReturnValue(mockProc.proc) + registerPtyHandlers(mainWindow as never) + const result = (await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24 + })) as { id: string } + restorePtyIncarnation(result.id, 'incarnation-old') + const text = ['x'.repeat(TERMINAL_INPUT_CHUNK_MAX_BYTES), 'tail'].join('') + + const writeResult = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id: result.id, + data: text + }) + expect(mockProc.proc.write).toHaveBeenCalledOnce() + restorePtyIncarnation(result.id, 'incarnation-new') + await vi.runAllTimersAsync() + + await expect(writeResult).resolves.toBe(false) + expect(mockProc.proc.write).toHaveBeenCalledOnce() + clearProviderPtyState(result.id) + }) + it('yields while validating accepted large acknowledged pty writes before provider writes', async () => { const mockProc = createMockProc() spawnMock.mockReturnValue(mockProc.proc) @@ -17684,6 +18005,30 @@ describe('registerPtyHandlers', () => { expect(mockProc.proc.write.mock.calls.map(([chunk]) => chunk).join('')).toBe(text) }) + it('drops a deferred write when the PTY incarnation changes during validation', async () => { + vi.useFakeTimers() + const mockProc = createMockProc() + spawnMock.mockReturnValue(mockProc.proc) + registerPtyHandlers(mainWindow as never) + const result = (await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24 + })) as { id: string } + restorePtyIncarnation(result.id, 'incarnation-old') + const text = 'é'.repeat(CLIPBOARD_TEXT_MEASURE_YIELD_CODE_UNITS + 1) + + const writeResult = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, { + id: result.id, + data: text + }) + restorePtyIncarnation(result.id, 'incarnation-new') + await vi.runAllTimersAsync() + + await expect(writeResult).resolves.toBe(false) + expect(mockProc.proc.write).not.toHaveBeenCalled() + clearProviderPtyState(result.id) + }) + it('rejects oversized acknowledged pty writes before provider writes', async () => { const mockProc = createMockProc() spawnMock.mockReturnValue(mockProc.proc) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index c2877303561..4b18f55f7e5 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -15,8 +15,8 @@ export { getBashShellReadyRcfileContent } from '../providers/local-pty-shell-rea import type { OrcaRuntimeService } from '../runtime/orca-runtime' import type { PtyBindingSourceExpectation, Store } from '../persistence' import { retireTerminalSurfaceFromPersistence } from '../runtime/mobile-session-terminal-persistence-retirement' +import { findTerminalTabIdForLeaf } from '../runtime/workspace-session-terminal-membership-authority' import type { GlobalSettings, TuiAgent } from '../../shared/types' -import { toSshExecutionHostId } from '../../shared/execution-host' import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' import { terminalOutputBacklogCapChars } from '../../shared/terminal-scrollback-policy' import type { @@ -84,6 +84,7 @@ import { import type { StartupCommandDelivery } from '../../shared/codex-startup-delivery' import { SSH_SESSION_EXPIRED_ERROR, + isSshPtyExitedError, isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors' @@ -234,7 +235,7 @@ import { } from '../project-groups/folder-workspace-path-status' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { resolveLocalProjectRuntimeForWorktreeId } from '../local-project-runtime-resolution' -import { isPtyIncarnationId } from '../../shared/pty-incarnation' +import { isPtyIncarnationId, isRelayAttestedPtyIncarnationId } from '../../shared/pty-incarnation' import type { PtyListedSession } from '../../shared/pty-listed-session' // ─── Provider Registry ────────────────────────────────────────────── @@ -286,6 +287,26 @@ const ptyPaneKey = new Map() // Why: reverse of ptyPaneKey — callers with a paneKey from outside the PTY lifecycle (e.g. agent-hook status routing) need the ptyId; kept in lock-step via the same sites. const paneKeyPtyId = new Map() +/** + * True only when this PTY's own pane has since bound a different PTY. The two + * maps above are maintained in lock-step, so their disagreement is proof the id + * was superseded — a renderer that queued work before a reattach cannot land it + * on the successor. + * + * Deliberately false when no paneKey is recorded: an unowned or orphaned PTY is + * unknown, not stale, and unknown never authorizes refusing an explicit + * operation. That is also what keeps orphan cleanup working, since those ids + * have no pane by construction. + */ +function isSupersededPtyId(ptyId: string): boolean { + const paneKey = ptyPaneKey.get(ptyId) + if (paneKey === undefined) { + return false + } + const currentPtyId = paneKeyPtyId.get(paneKey) + return currentPtyId !== undefined && currentPtyId !== ptyId +} + const AGENT_HOOK_RUNTIME_ENV_KEYS = [ 'ORCA_AGENT_HOOK_PORT', 'ORCA_AGENT_HOOK_TOKEN', @@ -515,6 +536,68 @@ function rememberPaneKeyForPty(ptyId: string, paneKey: unknown): string | null { return normalizedPaneKey } +/** + * The one producer of a pane -> shell binding: durable record and fence maps together, keyed by the + * tab holding the leaf *now* (a stored tabId names the tab a moved pane left). Splitting these let + * the superseded-PTY fence sit inert on reattach, the path it was built for. + * + * `bound` is false when a durable pane refuses. A throw is unknown rather than a refusal, so it + * propagates — only the caller that created the shell should clean it up. + * + * `tabId` is the resolved tab, returned so a caller that also registers the pane with the runtime + * graph uses the SAME coordinate. Registering under the lease's frozen tabId while the record and + * the fence use the live one splits the pane across two tabs, and the graph half then ensures a + * mobile surface for the tab the pane left. + */ +/** + * The tab a leaf lives in *now*. Callers whose own `tabId` is fresh (spawn) must not use this — the + * persisted layout is the stale side inside the renderer's publish debounce. Callers holding a + * tabId frozen in a durable lease (reattach) must, because the pane may have been moved since. + * + * Resolved separately from `bindPaneShell` so a thrown durable write cannot lose the answer and + * leave the runtime graph registered under the tab the pane left. + */ +export function resolvePaneShellTabId( + store: Pick | undefined, + leafId: string +): string | undefined { + if (typeof store?.getWorkspaceSession !== 'function') { + return undefined + } + return findTerminalTabIdForLeaf(store.getWorkspaceSession(), leafId) +} + +export function bindPaneShell(args: { + store: Pick | undefined + worktreeId: string + tabId: string + leafId: string + ptyId: string + incarnationId?: string + startupCwd?: string + mayCreate?: boolean + expectedBinding?: { ptyId: string; incarnationId?: string } + expectedSourceBinding?: PtyBindingSourceExpectation +}): { bound: boolean; tabId: string } { + const tabId = args.tabId + const bound = args.store?.persistPtyBinding({ + worktreeId: args.worktreeId, + tabId, + leafId: args.leafId, + ptyId: args.ptyId, + ...(args.incarnationId ? { incarnationId: args.incarnationId } : {}), + ...(args.startupCwd ? { startupCwd: args.startupCwd } : {}), + ...(args.mayCreate === false ? { mayCreate: false } : {}), + ...(args.expectedBinding ? { expectedBinding: args.expectedBinding } : {}), + ...(args.expectedSourceBinding ? { expectedSourceBinding: args.expectedSourceBinding } : {}) + }) + if (bound === false) { + return { bound: false, tabId } + } + rememberPaneKeyForPty(args.ptyId, makePaneKey(tabId, args.leafId)) + return { bound: true, tabId } +} + function cleanupPendingPaneSerializersForSender(ownerWebContentsId: number): void { pendingPaneSerializerCleanupRegistered.delete(ownerWebContentsId) for (const [paneKey, pending] of pendingByPaneKey) { @@ -651,11 +734,13 @@ type StablePaneAdoption = { } | null const stablePaneAdoptionsByOwnerKey = new Map>() +// Pane bindings have one home: the local partition. An SSH pane is not partitioned out — the +// renderer publishes its membership to `local` and `mayCreate: false` is evaluated there — so +// selecting `ssh:` here read a partition no live writer maintains (STA-3077 step P). function resolvePersistedStablePaneOwner( store: Store | undefined, paneKey: string, - worktreeId: string, - connectionId: string | null | undefined + worktreeId: string ): Pick | null { if (!store || typeof store.getWorkspaceSession !== 'function') { return null @@ -664,9 +749,7 @@ function resolvePersistedStablePaneOwner( if (!parsed) { return null } - const session = store.getWorkspaceSession( - connectionId ? toSshExecutionHostId(connectionId) : undefined - ) + const session = store.getWorkspaceSession() const tab = session.tabsByWorktree?.[worktreeId]?.find( (candidate) => candidate.id === parsed.tabId && candidate.worktreeId === worktreeId ) @@ -706,7 +789,7 @@ function resolveStablePaneOwner( } } } - const persisted = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) + const persisted = resolvePersistedStablePaneOwner(store, paneKey, worktreeId) if (resolved?.ptyId && persisted && resolved.ptyId !== persisted.ptyId) { throw new Error('terminal_pane_owner_conflict') } @@ -742,15 +825,13 @@ function resolveStablePaneOwner( function retirePersistedStablePaneOwner( store: Store | undefined, owner: StablePaneOwner, - worktreeId: string, - connectionId: string | null | undefined + worktreeId: string ): boolean { if (!store) { return false } const paneKey = makePaneKey(owner.tabId, owner.leafId) - const hostId = connectionId ? toSshExecutionHostId(connectionId) : undefined - const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) + const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId) if (!current) { // Why: persistence already dropped this pane binding (an earlier stop retired it while the // runtime kept history), so there is nothing left to clear — that is a completed retirement, @@ -760,7 +841,7 @@ function retirePersistedStablePaneOwner( if (current.ptyId !== owner.ptyId || current.incarnationId !== owner.persistedIncarnationId) { return false } - const session = store.getWorkspaceSession(hostId) + const session = store.getWorkspaceSession() const retired = retireTerminalSurfaceFromPersistence(session, { worktreeId, parentTabId: owner.tabId, @@ -771,7 +852,7 @@ function retirePersistedStablePaneOwner( if (retired === session) { return false } - store.setWorkspaceSession(retired, hostId) + store.setWorkspaceSession(retired) store.flushOrThrow() return true } @@ -786,6 +867,8 @@ type StablePaneSpawnContext = { connectionId?: string | null resolveOwner?: () => StablePaneOwner | null onFreshSpawn?: (result: PtySpawnResult) => void + /** Create a shell for this pane instead of attaching the one it records; see the caller. */ + refuseAdoption?: boolean } function stablePanePersistenceFence( @@ -805,25 +888,22 @@ function persistAdmittedStablePaneBinding(args: { result: PtySpawnResult worktreeId: string | undefined startupCwd: string | undefined - connectionId: string | null | undefined }): boolean { const expectedBinding = stablePanePersistenceFence(args.owner) if (!args.store || !args.owner || !args.worktreeId || !expectedBinding) { return false } - const persisted = args.store.persistPtyBinding( - { - worktreeId: args.worktreeId, - tabId: args.owner.tabId, - leafId: args.owner.leafId, - ptyId: args.result.id, - ...(args.result.incarnationId ? { incarnationId: args.result.incarnationId } : {}), - ...(args.startupCwd ? { startupCwd: args.startupCwd } : {}), - expectedBinding - }, - args.connectionId ? toSshExecutionHostId(args.connectionId) : undefined - ) - if (persisted === false) { + const persisted = bindPaneShell({ + store: args.store, + worktreeId: args.worktreeId, + tabId: args.owner.tabId, + leafId: args.owner.leafId, + ptyId: args.result.id, + ...(args.result.incarnationId ? { incarnationId: args.result.incarnationId } : {}), + ...(args.startupCwd ? { startupCwd: args.startupCwd } : {}), + expectedBinding + }) + if (persisted.bound === false) { throw new Error('terminal_pane_owner_changed') } return true @@ -859,7 +939,7 @@ async function attachStablePaneOwner( if (isDaemonEndpointGoneError(error)) { throw new TerminalHostGoneError() } - if (!isPtyAlreadyGoneError(error)) { + if (!isPtyProvenGoneForReplacement(error, owner.ptyId)) { throw error } const ownerBeforeRetire = args.resolveOwner?.() @@ -875,10 +955,7 @@ async function attachStablePaneOwner( runtime?.onPtyExit(owner.ptyId, 0, owner.incarnationId) clearProviderPtyState(owner.ptyId) ptyOwnership.delete(owner.ptyId) - if ( - args.worktreeId && - !retirePersistedStablePaneOwner(args.store, owner, args.worktreeId, args.connectionId) - ) { + if (args.worktreeId && !retirePersistedStablePaneOwner(args.store, owner, args.worktreeId)) { throw new Error('terminal_pane_owner_changed') } if (args.resolveOwner?.()) { @@ -901,7 +978,11 @@ async function attachStablePaneOwner( async function spawnForStablePane( args: StablePaneSpawnContext ): Promise<{ result: PtySpawnResult; owner: StablePaneOwner | null }> { - if (args.owner) { + // The one place an owner becomes `sessionId` for the provider, which is what makes an attach an + // attach. Refusing adoption has to be honoured HERE: gating only where the owner is resolved + // leaves every other resolution free to reach this line, which is exactly how the unreachable + // pane's "start a new terminal" kept attaching the shell it could not reach. + if (args.owner && !args.refuseAdoption) { const attached = await attachStablePaneOwner({ ...args, owner: args.owner }) if (attached) { return attached @@ -1028,9 +1109,29 @@ function normalizeNodePtySpawnError(err: unknown): Error { return new Error(hintedMessage) } +/** + * Retiring a pane's owner authorizes a replacement that carries its agent resume payload, so it + * needs proof — unlike a shutdown, where "not found" is simply the outcome we asked for. A bare + * not-found proves an exit only from a provider that owns its ptys; a replacement relay answers the + * same for shells its predecessor is still running. + */ +function isPtyProvenGoneForReplacement(err: unknown, ptyId: string): boolean { + if (!parseAppSshPtyId(ptyId)) { + return isPtyAlreadyGoneError(err) + } + // For SSH, the two proving answers are the relay's own observed exit and the expiry the reattach + // mints only after verifying that proof names this shell. A bare not-found is neither. + const message = err instanceof Error ? err.message : String(err) + return isSshPtyExitedError(err) || message.includes(SSH_SESSION_EXPIRED_ERROR) +} + function isPtyAlreadyGoneError(err: unknown): boolean { const message = err instanceof Error ? err.message : String(err) - return isSshPtyNotFoundError(err) || /Session not found/i.test(message) + // A proven exit is the strongest form of "already gone"; it only reads differently because the + // relay now says what it observed instead of reporting an unknown id. + return ( + isSshPtyNotFoundError(err) || isSshPtyExitedError(err) || /Session not found/i.test(message) + ) } function delay(ms: number): Promise { @@ -5132,8 +5233,7 @@ export function registerPtyHandlers( owner: stablePaneOwner, result, worktreeId: hostSessionBinding?.worktreeId, - startupCwd: cwd, - connectionId: args.connectionId + startupCwd: cwd }) } catch (error) { if (error instanceof Error && error.message === 'terminal_pane_owner_changed') { @@ -5201,6 +5301,12 @@ export function registerPtyHandlers( ...(typeof args.leafId === 'string' && isTerminalLeafId(args.leafId) ? { leafId: args.leafId } : {}), + // The shell this lease names, as the host just attested it. Without it the record says + // only "pty-N", and a replaced relay reissues that from 1 — so reconnect would have + // nothing to compare and would attach whatever now answers to the id. + ...(isRelayAttestedPtyIncarnationId(result.incarnationId) + ? { incarnationId: result.incarnationId } + : {}), state: 'attached', lastAttachedAt: Date.now() }) @@ -5224,7 +5330,8 @@ export function registerPtyHandlers( }) if (hostSessionBinding && !stablePaneBindingPersisted) { try { - const binding = { + const bound = bindPaneShell({ + store: hostSessionBinding.store, worktreeId: hostSessionBinding.worktreeId, tabId: hostSessionBinding.tabId, leafId: hostSessionBinding.leafId, @@ -5234,14 +5341,8 @@ export function registerPtyHandlers( ...(hostSessionBinding.expectedSourceBinding ? { expectedSourceBinding: hostSessionBinding.expectedSourceBinding } : {}) - } - const persisted = args.connectionId - ? hostSessionBinding.store.persistPtyBinding( - binding, - toSshExecutionHostId(args.connectionId) - ) - : hostSessionBinding.store.persistPtyBinding(binding) - if (persisted === false) { + }) + if (bound.bound === false) { throw new Error('terminal_split_source_not_found') } } catch (err) { @@ -5845,6 +5946,8 @@ export function registerPtyHandlers( cwd?: string // Why: fresh local spawns opt into recovering a saved cwd whose dir was deleted (#7239); reattach/remote need exact cwd, so the flag alone isn't sufficient. cwdFallback?: 'worktree' + /** Create a shell for this pane rather than adopting the one it records. */ + createFreshShellForUnreachablePane?: boolean env?: Record envToDelete?: string[] command?: string @@ -5944,8 +6047,14 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return { ...(await existingPaneSpawn.promise), isReattach: true } } + // Why the caller may refuse adoption: the unreachable-pane card offers "Start a new terminal" + // for a pane whose recorded shell cannot be reached. Resolving an owner here makes the action + // attach that shell first — and when it is unreachable the attach fails, the action creates + // nothing, and the card comes back. The button is dead in the one state it is offered in. + // Skipping the resolve is what makes it a creation: nothing is killed, and the old shell is + // left alive and unbound for the cleanup surface. const earlyStablePaneOwner = - earlyPaneKey && args.worktreeId + earlyPaneKey && args.worktreeId && args.createFreshShellForUnreachablePane !== true ? resolveStablePaneOwner(runtime, store, earlyPaneKey, args.worktreeId, args.connectionId) : null const earlyWorktreeId = args.worktreeId @@ -6555,6 +6664,7 @@ export function registerPtyHandlers( provider, spawnOptions, owner: stablePaneOwnerCandidate, + refuseAdoption: args.createFreshShellForUnreachablePane === true, worktreeId: args.worktreeId, connectionId: args.connectionId, resolveOwner: () => @@ -6694,8 +6804,7 @@ export function registerPtyHandlers( owner: stablePaneOwner, result, worktreeId: args.worktreeId, - startupCwd: cwd, - connectionId: args.connectionId + startupCwd: cwd }) } catch (error) { if (error instanceof Error && error.message === 'terminal_pane_owner_changed') { @@ -6748,6 +6857,11 @@ export function registerPtyHandlers( ...(typeof args.worktreeId === 'string' ? { worktreeId: args.worktreeId } : {}), ...(typeof args.tabId === 'string' ? { tabId: args.tabId } : {}), ...(validatedLeafId ? { leafId: validatedLeafId } : {}), + // See the sibling writer: a lease that names only "pty-N" cannot survive a relay that + // reissues ids from 1. + ...(isRelayAttestedPtyIncarnationId(result.incarnationId) + ? { incarnationId: result.incarnationId } + : {}), state: 'attached', lastAttachedAt: Date.now() }) @@ -6768,19 +6882,15 @@ export function registerPtyHandlers( !stablePaneBindingPersisted ) { try { - const binding = { + bindPaneShell({ + store, worktreeId: args.worktreeId, tabId: args.tabId, leafId: validatedLeafId, ptyId: result.id, ...(result.incarnationId ? { incarnationId: result.incarnationId } : {}), ...(cwd ? { startupCwd: cwd } : {}) - } - if (args.connectionId) { - store.persistPtyBinding(binding, toSshExecutionHostId(args.connectionId)) - } else { - store.persistPtyBinding(binding) - } + }) } catch (err) { console.error('[pty] failed to persist PTY binding after spawn:', err) if (!result.isReattach) { @@ -7049,11 +7159,20 @@ export function registerPtyHandlers( mainWindow.webContents.send('pty:writeUnavailable', { id }) } + type PtyWriteFence = { incarnationId: string | undefined } + + const isCurrentPtyWrite = (id: string, fence: PtyWriteFence): boolean => + !isSupersededPtyId(id) && ptyIncarnationById.get(id) === fence.incarnationId + const writePtyProviderInputWithinLimit = ( provider: IPtyProvider, id: string, - data: string + data: string, + fence: PtyWriteFence ): boolean | Promise => { + if (!isCurrentPtyWrite(id, fence)) { + return false + } const chunks = iterateTerminalInputChunks(data) const first = chunks.next() if (first.done) { @@ -7065,21 +7184,24 @@ export function registerPtyHandlers( provider.write(id, first.value) return true } - return writePtyProviderInputChunks(provider, id, chunks, first.value, second.value) + return writePtyProviderInputChunks(provider, id, chunks, first.value, second.value, fence) } const writePtyProviderInput = ( provider: IPtyProvider, id: string, - data: string + data: string, + fence: PtyWriteFence ): boolean | Promise => { try { const tooLarge = isTerminalInputTooLargeWithDeferredMeasurement(data) if (typeof tooLarge === 'boolean') { - return tooLarge ? false : writePtyProviderInputWithinLimit(provider, id, data) + return tooLarge ? false : writePtyProviderInputWithinLimit(provider, id, data, fence) } return tooLarge - .then((result) => (result ? false : writePtyProviderInputWithinLimit(provider, id, data))) + .then((result) => + result ? false : writePtyProviderInputWithinLimit(provider, id, data, fence) + ) .catch((error) => { reportUnavailablePtyWrite(id, error) return false @@ -7095,12 +7217,16 @@ export function registerPtyHandlers( id: string, chunks: Iterator, firstChunk: string, - secondChunk: string + secondChunk: string, + fence: PtyWriteFence ): Promise => { try { let chunk: IteratorResult = { done: false, value: firstChunk } let nextChunk: IteratorResult = { done: false, value: secondChunk } while (!chunk.done) { + if (!isCurrentPtyWrite(id, fence)) { + return false + } provider.write(id, chunk.value) if (!nextChunk.done) { await new Promise((resolve) => setTimeout(resolve, 0)) @@ -7145,11 +7271,17 @@ export function registerPtyHandlers( !mainWindow.isDestroyed() && !(typeof mainWebContents.isDestroyed === 'function' && mainWebContents.isDestroyed()) - const writePtyInput = (args: PtyWritePayload): boolean | Promise => { + const writePtyInput = ( + args: PtyWritePayload, + fence: PtyWriteFence + ): boolean | Promise => { // Why: mobile-presence-lock defense-in-depth — the renderer's onData guard can let one keystroke slip during the state-flip lag, so catch it server-side. See docs/mobile-presence-lock.md. if (runtime?.getDriver(args.id).kind === 'mobile') { return false } + if (!isCurrentPtyWrite(args.id, fence)) { + return false + } const provider = ptyOwnership.has(args.id) ? tryGetProviderForPty(args.id) : undefined if (!provider) { return false @@ -7161,16 +7293,22 @@ export function registerPtyHandlers( if (visibleRendererPtys.has(args.id)) { clearHiddenRendererResizeOutput(args.id) } - return writePtyProviderInput(provider, args.id, args.data) + return writePtyProviderInput(provider, args.id, args.data, fence) } catch { return false } } - const writePtyInputAccepted = (args: PtyWritePayload): boolean | Promise => { + const writePtyInputAccepted = ( + args: PtyWritePayload, + fence: PtyWriteFence + ): boolean | Promise => { if (runtime?.getDriver(args.id).kind === 'mobile') { return false } + if (!isCurrentPtyWrite(args.id, fence)) { + return false + } // Why: the ack infers Ctrl+C/Escape reached the local PTY; SSH providers are fire-and-forget relay notifications and can't truthfully acknowledge yet. if (ptyOwnership.get(args.id) !== null) { return false @@ -7186,7 +7324,7 @@ export function registerPtyHandlers( if (visibleRendererPtys.has(args.id)) { clearHiddenRendererResizeOutput(args.id) } - return writePtyProviderInput(provider, args.id, args.data) + return writePtyProviderInput(provider, args.id, args.data, fence) } catch { return false } @@ -7198,21 +7336,31 @@ export function registerPtyHandlers( if (!isPtyWriteEventFromMainWindow(event, mainWindow.webContents) || !isPtyWritePayload(args)) { return } - const claimTail = hostViewportClaimTails.get(args.id) - if (claimTail) { - void claimTail.then((claimed) => (claimed ? writePtyInput(args) : false)) + // Why here and not in the renderer: input queued before a reattach would + // otherwise land on whatever PTY now holds the pane. + const fence: PtyWriteFence = { incarnationId: ptyIncarnationById.get(args.id) } + if (!isCurrentPtyWrite(args.id, fence)) { return } - writePtyInput(args) + const claimTail = hostViewportClaimTails.get(args.id) + if (claimTail) { + void claimTail.then((claimed) => (claimed ? writePtyInput(args, fence) : false)) + return + } + writePtyInput(args, fence) }) ipcMain.handle('pty:writeAccepted', (event, args: unknown): boolean | Promise => { if (!isPtyWriteEventFromMainWindow(event, mainWindow.webContents) || !isPtyWritePayload(args)) { return false } + const fence: PtyWriteFence = { incarnationId: ptyIncarnationById.get(args.id) } + if (!isCurrentPtyWrite(args.id, fence)) { + return false + } const claimTail = hostViewportClaimTails.get(args.id) return claimTail - ? claimTail.then((claimed) => (claimed ? writePtyInputAccepted(args) : false)) - : writePtyInputAccepted(args) + ? claimTail.then((claimed) => (claimed ? writePtyInputAccepted(args, fence) : false)) + : writePtyInputAccepted(args, fence) }) ipcMain.removeAllListeners('pty:claimViewport') @@ -7245,6 +7393,10 @@ export function registerPtyHandlers( // Why: resize is fire-and-forget — ipcMain.on (not .handle) halves IPC traffic by skipping the empty acknowledgement reply. ipcMain.removeAllListeners('pty:resize') ipcMain.on('pty:resize', (_event, args: { id: string; cols: number; rows: number }) => { + // Why: a resize for a pane that has rebound would reshape the successor's shell. + if (isSupersededPtyId(args.id)) { + return + } // Why: after a desktop-fit override change the renderer's safeFit cascade re-measures ALL panes (background ones at full width), so suppress every pty:resize in this window to avoid corrupting PTY dimensions. if (runtime?.isResizeSuppressed()) { return @@ -7509,6 +7661,12 @@ export function registerPtyHandlers( ipcMain.removeAllListeners('pty:signal') ipcMain.on('pty:signal', (_event, args: { id: string; signal: string }) => { + // Why fenced but pty:kill is not: a signal means "interrupt MY pane", so a + // superseded id is a misdirected interrupt. A kill on a superseded id is the + // opposite — that PTY is now orphaned and reclaiming it is the point. + if (isSupersededPtyId(args.id)) { + return + } tryGetProviderForPty(args.id) ?.sendSignal(args.id, args.signal) .catch(() => {}) @@ -7661,6 +7819,9 @@ export function registerPtyHandlers( } const ownedConnectionId = ptyOwnership.get(args.id) const parsedSshId = ownedConnectionId === undefined ? parseAppSshPtyId(args.id) : null + // Why: before the cold-start daemon swap lands, the pre-swap local provider owns no + // daemon id and would answer an authoritative false for every restored session. + await getLocalPtyProviderStartupPromise(ownedConnectionId ?? parsedSshId?.connectionId) const provider = parsedSshId ? sshProviders.get(parsedSshId.connectionId) : tryGetProviderForPty(args.id) diff --git a/src/main/ipc/ssh-pty-output-intake.test.ts b/src/main/ipc/ssh-pty-output-intake.test.ts index be38872da04..43ceed26079 100644 --- a/src/main/ipc/ssh-pty-output-intake.test.ts +++ b/src/main/ipc/ssh-pty-output-intake.test.ts @@ -3,7 +3,7 @@ import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' import { createSshPtyOutputIntakeHarness as createHarness, sshPtyOutputEvent as event -} from './ssh-pty-output-intake-test-harness' +} from './__tests__/ssh-pty-output-intake-test-harness' describe('SshPtyOutputIntake', () => { it('plateaus at the model and pressure budgets, then resumes below low water', async () => { diff --git a/src/main/ipc/ssh-pty-output-model-migration.test.ts b/src/main/ipc/ssh-pty-output-model-migration.test.ts index ae56e2e45fc..59a063fba02 100644 --- a/src/main/ipc/ssh-pty-output-model-migration.test.ts +++ b/src/main/ipc/ssh-pty-output-model-migration.test.ts @@ -3,7 +3,7 @@ import type { SshPtyOutputDataEvent } from './ssh-pty-output-intake' import { createSshPtyOutputIntakeHarness as createHarness, sshPtyOutputEvent as event -} from './ssh-pty-output-intake-test-harness' +} from './__tests__/ssh-pty-output-intake-test-harness' // Production-shaped ids: bare relay id on the wire side, prefixed app id in intake. const APP_ID = 'ssh:conn@@pty-1' diff --git a/src/main/ipc/ssh.test.ts b/src/main/ipc/ssh.test.ts index 538cf4bf846..d4dcde25e4e 100644 --- a/src/main/ipc/ssh.test.ts +++ b/src/main/ipc/ssh.test.ts @@ -305,6 +305,7 @@ describe('SSH IPC handlers', () => { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), diff --git a/src/main/persistence-ssh-lease-termination.test.ts b/src/main/persistence-ssh-lease-termination.test.ts new file mode 100644 index 00000000000..c1c70c5c667 --- /dev/null +++ b/src/main/persistence-ssh-lease-termination.test.ts @@ -0,0 +1,152 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { getDefaultPersistedState } from '../shared/constants' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('./telemetry/client', () => ({ track: vi.fn() })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TARGET = 'ssh-target-1' + +type DurableFlushStore = { + flushDurableStateOrThrowAsync: (drainToStableGeneration?: boolean) => Promise +} + +function lease(ptyId: string, state: 'attached' | 'detached') { + return { + targetId: TARGET, + ptyId, + state, + createdAt: 1, + updatedAt: 1 + } +} + +async function createStore() { + mkdirSync(testState.dir, { recursive: true }) + writeFileSync( + join(testState.dir, 'orca-data.json'), + JSON.stringify({ + ...getDefaultPersistedState(testState.dir), + sshRemotePtyLeases: [ + lease('pty-1', 'detached'), + lease('pty-2', 'attached'), + lease('pty-3', 'detached') + ] + }), + 'utf-8' + ) + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() +} + +beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-ssh-lease-termination-')) +}) + +afterEach(() => { + rmSync(testState.dir, { force: true, recursive: true }) +}) + +describe('SSH lease termination persistence', () => { + it('updates only selected leases and awaits one asynchronous durability barrier', async () => { + const store = await createStore() + const asyncFlush = vi + .spyOn(store as unknown as DurableFlushStore, 'flushDurableStateOrThrowAsync') + .mockResolvedValue() + const syncFlush = vi.spyOn(store, 'flush') + const syncFlushOrThrow = vi.spyOn(store, 'flushOrThrow') + + await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1', 'pty-3']) + + expect(asyncFlush).toHaveBeenCalledOnce() + expect(asyncFlush).toHaveBeenCalledWith(false) + expect(syncFlush).not.toHaveBeenCalled() + expect(syncFlushOrThrow).not.toHaveBeenCalled() + expect(store.getSshRemotePtyLeases(TARGET)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }), + expect.objectContaining({ ptyId: 'pty-2', state: 'attached' }), + expect.objectContaining({ ptyId: 'pty-3', state: 'terminated' }) + ]) + ) + }) + + it('does not overwrite concurrent lease state when durability fails', async () => { + const store = await createStore() + const writeError = new Error('disk full') + const asyncFlush = vi + .spyOn(store as unknown as DurableFlushStore, 'flushDurableStateOrThrowAsync') + .mockRejectedValue(writeError) + const scheduleSave = vi.spyOn(store as unknown as { scheduleSave: () => void }, 'scheduleSave') + scheduleSave.mockClear() + + const retirement = store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1']) + store.markSshRemotePtyLeasesForShutdown(TARGET, 'detached') + + await expect(retirement).rejects.toBe(writeError) + expect(scheduleSave).toHaveBeenCalledTimes(2) + expect(store.getSshRemotePtyLeases(TARGET)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }), + expect.objectContaining({ ptyId: 'pty-2', state: 'detached' }) + ]) + ) + + asyncFlush.mockRestore() + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + const persisted = JSON.parse(readFileSync(join(testState.dir, 'orca-data.json'), 'utf-8')) as { + sshRemotePtyLeases: { ptyId: string; state: string }[] + } + expect(persisted.sshRemotePtyLeases).toContainEqual( + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }) + ) + }) + + it('does not write when no requested lease exists', async () => { + const store = await createStore() + const asyncFlush = vi.spyOn( + store as unknown as DurableFlushStore, + 'flushDurableStateOrThrowAsync' + ) + + await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['missing']) + + expect(asyncFlush).not.toHaveBeenCalled() + }) + + it('does not chase mutations made after the retirement generation is durable', async () => { + const store = await createStore() + const internal = store as unknown as { enqueueWrite: () => Promise } + const enqueueWrite = internal.enqueueWrite.bind(store) + let injectedMutation = false + const enqueueSpy = vi.spyOn(internal, 'enqueueWrite').mockImplementation(async () => { + await enqueueWrite() + if (!injectedMutation) { + injectedMutation = true + store.updateUI({ sidebarWidth: 777 }) + } + }) + + await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1']) + + expect(enqueueSpy).toHaveBeenCalledOnce() + const persisted = JSON.parse(readFileSync(join(testState.dir, 'orca-data.json'), 'utf-8')) as { + sshRemotePtyLeases: { ptyId: string; state: string }[] + } + expect(persisted.sshRemotePtyLeases).toContainEqual( + expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }) + ) + enqueueSpy.mockRestore() + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) +}) diff --git a/src/main/persistence.test.ts b/src/main/persistence.test.ts index 17a73fa0e48..899aaa001f8 100644 --- a/src/main/persistence.test.ts +++ b/src/main/persistence.test.ts @@ -9992,6 +9992,9 @@ describe('Store', () => { expect(store.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey?.[paneKey]).toBe( 'inc-live' ) + // Unchanged by STA-3077 step P. The fold moves an SSH pane's incarnation only when it moves + // the binding it fences, and this fixture's tab exists solely in the SSH partition — so + // there is no second home to disagree with, nothing folds, and both halves stay together. const reloaded = await createStore() expect(reloaded.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey?.[paneKey]).toBe( 'inc-live' diff --git a/src/main/persistence.ts b/src/main/persistence.ts index f9fee511d39..14d50d4d297 100644 --- a/src/main/persistence.ts +++ b/src/main/persistence.ts @@ -37,6 +37,7 @@ import { nextAutomationOccurrenceAfter } from '../shared/automation-schedules' import { getAutomationLegacyRepoId } from '../shared/automation-run-identity' +import { isRelayAttestedPtyIncarnationId } from '../shared/pty-incarnation' import { normalizeAutomationPrecheck } from '../shared/automation-precheck' import { normalizeProxyUrl } from '../shared/network-proxy' import { normalizeKagiSessionLink } from '../shared/browser-url' @@ -96,6 +97,7 @@ import type { MigrationUnsupportedPtyEntry } from '../shared/agent-status-types' import { MOBILE_PAIRING_USERDATA_FILES } from './runtime/mobile-pairing-files' import { normalizePersistedMobileClientTabSelections } from './runtime/client-session-tab-selection-persistence' import { sanitizeWorkspaceSessionTerminalRetirements } from './runtime/mobile-session-terminal-persistence-retirement' +import { findTerminalTabIdForLeaf } from './runtime/workspace-session-terminal-membership-authority' import { removeRepoFromHostWorkspaceSessions, removeRepoFromWorkspaceSession @@ -1632,7 +1634,13 @@ function normalizeSshRemotePtyLease(value: unknown): SshRemotePtyLease | null { createdAt: typeof raw.createdAt === 'number' ? raw.createdAt : now, updatedAt: typeof raw.updatedAt === 'number' ? raw.updatedAt : now, ...(typeof raw.lastAttachedAt === 'number' ? { lastAttachedAt: raw.lastAttachedAt } : {}), - ...(typeof raw.lastDetachedAt === 'number' ? { lastDetachedAt: raw.lastDetachedAt } : {}) + ...(typeof raw.lastDetachedAt === 'number' ? { lastDetachedAt: raw.lastDetachedAt } : {}), + // Rebuilt field by field, so a lease is only as durable as this list — adding the field to the + // type without adding it here drops it on every load. Synthesized values are refused rather + // than carried: one written by an older build would later read as a different shell. + ...(isRelayAttestedPtyIncarnationId(raw.incarnationId) + ? { incarnationId: raw.incarnationId } + : {}) } } @@ -1733,6 +1741,20 @@ function layoutContainsLeafId(node: TerminalPaneLayoutNode | null, leafId: strin return layoutContainsLeafId(node.first, leafId) || layoutContainsLeafId(node.second, leafId) } +/** Total order so two leases for one pane resolve the same way on every host. */ +function isNewerSshRemotePtyLease( + candidate: SshRemotePtyLease, + incumbent: SshRemotePtyLease +): boolean { + if (candidate.updatedAt !== incumbent.updatedAt) { + return candidate.updatedAt > incumbent.updatedAt + } + if (candidate.createdAt !== incumbent.createdAt) { + return candidate.createdAt > incumbent.createdAt + } + return candidate.ptyId > incumbent.ptyId +} + function cloneLayoutNode(node: TerminalPaneLayoutNode): TerminalPaneLayoutNode { if (node.type === 'leaf') { return { type: 'leaf', leafId: node.leafId } @@ -2828,6 +2850,7 @@ export class Store { private pendingGithubCacheWrite: Promise | null = null private readonly staleGithubCacheTempCleanup: Promise private gitUsernameCache = new Map() + private readonly sshRemotePtyLeaseMutationVersions = new WeakMap() private readonly protectedSecrets = new ProtectedSecretPersistence() private loadNeedsSave = false private settingsChangeListeners = new Set< @@ -6841,10 +6864,14 @@ export class Store { incarnationId?: string startupCwd?: string expectedBinding?: { ptyId: string; incarnationId?: string } + /** Reattach passes false: an absent durable pane never authorizes creating UI. + * Defaults true so the spawn path keeps its force-quit-race branches. */ + mayCreate?: boolean expectedSourceBinding?: PtyBindingSourceExpectation }, hostId?: string | null ): boolean { + const mayCreate = args.mayCreate ?? true const resolvedHostId = this.resolveHostId(hostId) const session = this.getWorkspaceSession(resolvedHostId) const paneKey = `${args.tabId}:${args.leafId}` @@ -6957,6 +6984,10 @@ export class Store { [bindingWorktreeId]: session.activeTabIdByWorktree?.[bindingWorktreeId] ?? args.tabId } } + if (!mayCreate && terminalMembershipChanged) { + restoreSession() + return false + } if (!isTerminalLeafId(args.leafId)) { // Why: keep legacy renderer-local pane ids out of durable leaf-keyed layout state after the UUID migration. advanceTopologyFence() @@ -7007,6 +7038,10 @@ export class Store { } } } + if (!mayCreate && terminalMembershipChanged) { + restoreSession() + return false + } advanceTopologyFence() try { this.flushOrThrow() @@ -7302,6 +7337,12 @@ export class Store { return leases.filter((lease) => targetId === undefined || lease.targetId === targetId) } + private advanceSshRemotePtyLeaseMutationVersion(lease: SshRemotePtyLease): number { + const version = (this.sshRemotePtyLeaseMutationVersions.get(lease) ?? 0) + 1 + this.sshRemotePtyLeaseMutationVersions.set(lease, version) + return version + } + upsertSshRemotePtyLease( lease: Omit & Partial> @@ -7333,9 +7374,301 @@ export class Store { } else { this.state.sshRemotePtyLeases.push(next) } + this.supersedeSiblingLeasesForPane(next, now) this.flush() } + /** + * One pane owns at most one live remote PTY. Without this, lease identity is + * `(targetId, ptyId)` alone, so a pane that re-leases under a new relay id + * leaves its predecessor live forever — reattach then fans out over both and + * grafts a pane the user never opened (STA-3077: 2 -> 19 -> 20 across three + * reconnects). + * + * Superseded leases are marked `expired`, not terminated: the remote shell is + * deliberately left running, because losing a lease is not proof the shell died. + */ + private supersedeSiblingLeasesForPane(winner: SshRemotePtyLease, now: number): void { + if (!winner.worktreeId || !winner.tabId || !winner.leafId) { + return + } + if (winner.state === 'terminated' || winner.state === 'expired') { + return + } + // Why consult the binding here: at upsert time the caller's lease may not be + // the one the pane is bound to yet. Expiring the bound predecessor would + // detach a live pane, so leave both live and let reattach arbitrate with the + // binding in hand. + const boundPtyId = this.durablyBoundPtyIdForPane(winner.targetId, winner.tabId, winner.leafId) + if (boundPtyId && boundPtyId !== winner.ptyId) { + return + } + const superseded: SshRemotePtyLease[] = [] + for (const lease of this.state.sshRemotePtyLeases ?? []) { + if ( + lease.ptyId !== winner.ptyId && + lease.targetId === winner.targetId && + lease.worktreeId === winner.worktreeId && + // Leaf only: a lease freezes its tabId, and a pane broken out to a new tab would otherwise + // never compete with its own predecessor — which is the reported cardinality growth. + lease.leafId === winner.leafId + ) { + if (lease.state === 'expired') { + // A concurrent same-pane owner confirms this lease must outlive rollback as retired. + this.advanceSshRemotePtyLeaseMutationVersion(lease) + continue + } + if (lease.state === 'terminated') { + continue + } + lease.state = 'expired' + lease.updatedAt = now + this.advanceSshRemotePtyLeaseMutationVersion(lease) + superseded.push(lease) + } + } + // Why: matching on lease ptyId first means this scrubs only the predecessor's + // stale binding — the winner's own binding cannot match and is left intact. + this.clearSshRemotePtyBindingsForLeases(winner.targetId, superseded, 'local') + } + + /** The PTY a pane is durably bound to. The desktop plane's home is `local` — the renderer is its + * only publisher of pane membership. Hedging into `ssh:` let the headless plane's copy + * outvote the live binding and silently no-op supersession (STA-3077). + * + * Keyed on the leaf, falling back across tabs: a lease freezes its tabId at write time, and + * `detachTerminalPaneToTab` moves a live pane, so the named tab can be the one the pane left. + * Missing the binding there makes it lose to recency and retires the pane's own shell. */ + private durablyBoundPtyIdForPane( + targetId: string, + tabId: string, + leafId: string + ): string | undefined { + // Deliberately does NOT require the tab to still exist, unlike findTerminalTabIdForLeaf, which + // must — binding a live shell to a deleted tab registers a ghost pane. Here the consequence + // runs the other way: a binding under a departed tab only ever keeps a lease alive, and not + // retiring is the safe direction when nothing has proved the shell dead. + const findLeafBinding = (session: WorkspaceSessionState | undefined): string | undefined => { + const layouts = session?.terminalLayoutsByTabId + return ( + layouts?.[tabId]?.ptyIdsByLeafId?.[leafId] ?? + Object.values(layouts ?? {}).find((layout) => layout?.ptyIdsByLeafId?.[leafId]) + ?.ptyIdsByLeafId?.[leafId] + ) + } + // Local FIRST, and only then the headless plane. Local-first is what STA-3077 needed: the old + // code preferred `ssh:`, so a copy no live writer maintained outvoted the real binding + // and supersession no-opped. The fallback is not that hedge — it only speaks for a pane local + // says nothing about, so a headless-owned pane still gets a vote before its lease is retired. + const boundPtyId = + findLeafBinding(this.state.workspaceSession) ?? + findLeafBinding(this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)]) + return boundPtyId ? this.getRelayPtyIdForSshLeaseComparison(targetId, boundPtyId) : undefined + } + + /** + * The durable pane binding outranks recency. Picking the newest lease alone + * would retire the one the pane is actually bound to whenever a newer unbound + * lease exists, which detaches a live pane instead of healing it. + */ + private outranksForPane( + candidate: SshRemotePtyLease, + incumbent: SshRemotePtyLease, + targetId: string + ): boolean { + const boundPtyId = this.durablyBoundPtyIdForPane( + targetId, + candidate.tabId ?? '', + candidate.leafId ?? '' + ) + if (boundPtyId) { + if (incumbent.ptyId === boundPtyId) { + return false + } + if (candidate.ptyId === boundPtyId) { + return true + } + } + return isNewerSshRemotePtyLease(candidate, incumbent) + } + + /** + * Heals lease state that predates pane-keyed supersession. Existing installs + * carry the duplicates STA-3077 accumulated — one report reached 20 live + * leases for a handful of panes — and supersession alone only prevents new + * ones. Reattach calls this first so a stale predecessor cannot be revived. + * + * Returns the number of leases retired, for logging. + */ + async supersedeDuplicatePaneLeases(targetId: string): Promise { + const live = (this.state.sshRemotePtyLeases ?? []).filter( + (lease) => + lease.targetId === targetId && lease.state !== 'terminated' && lease.state !== 'expired' + ) + const winnerByPane = new Map() + for (const lease of live) { + if (!lease.worktreeId || !lease.tabId || !lease.leafId) { + continue + } + // Keyed on the leaf, not the frozen tabId: the same pane moved between tabs must land in one + // bucket, or its duplicates never arbitrate against each other. + const paneKey = [lease.worktreeId, lease.leafId].join('\0') + const incumbent = winnerByPane.get(paneKey) + if (!incumbent || this.outranksForPane(lease, incumbent, targetId)) { + winnerByPane.set(paneKey, lease) + } + } + if (winnerByPane.size === 0) { + return 0 + } + const winners = new Set(winnerByPane.values()) + const now = Date.now() + const superseded: SshRemotePtyLease[] = [] + const restore: (() => void)[] = [] + for (const lease of live) { + if (!lease.worktreeId || !lease.tabId || !lease.leafId || winners.has(lease)) { + continue + } + const { state, updatedAt } = lease + lease.state = 'expired' + lease.updatedAt = now + const rollbackVersion = this.advanceSshRemotePtyLeaseMutationVersion(lease) + restore.push(() => { + if ( + this.state.sshRemotePtyLeases?.includes(lease) && + this.sshRemotePtyLeaseMutationVersions.get(lease) === rollbackVersion && + lease.state === 'expired' && + lease.updatedAt === now + ) { + lease.state = state + lease.updatedAt = updatedAt + this.advanceSshRemotePtyLeaseMutationVersion(lease) + } + }) + superseded.push(lease) + } + if (superseded.length === 0) { + return 0 + } + const bindingsBefore = this.snapshotSshLeaseBindings(targetId, superseded) + this.clearSshRemotePtyBindingsForLeases(targetId, superseded, 'local') + this.scheduleSave() + try { + // Why the ASYNC twin: the sync flush fsyncs a multi-MB file from the Electron main thread, + // and this runs on every reconnect. On a stalled network profile mount that syscall is + // uninterruptible and nothing can bound it — see flushAsync. Why "OrThrow" and not plain + // flush(): flush() swallows write errors, which would leave these leases retired in memory + // but attached on disk for the rest of the session. + await this.flushDurableStateOrThrowAsync(false) + } catch (err) { + for (const undo of restore) { + undo() + } + this.restoreSshLeaseBindings(targetId, superseded, bindingsBefore) + this.scheduleSave() + console.error('[persistence] Failed to retire duplicate pane leases:', err) + return 0 + } + return superseded.length + } + + private snapshotSshLeaseBindings( + targetId: string, + leases: SshRemotePtyLease[] + ): { + tabs: { leafId: string; ptyId: string }[] + leaves: { leafId: string; ptyId: string }[] + } { + const tabSnapshots: { leafId: string; ptyId: string }[] = [] + const leaves: { leafId: string; ptyId: string }[] = [] + const session = this.state.workspaceSession + for (const [worktreeId, tabs] of Object.entries(session?.tabsByWorktree ?? {})) { + for (const tab of tabs) { + if (tab.ptyId) { + const lease = leases.find((candidate) => + this.sshRemotePtyLeaseMayReferenceBinding(candidate, { + ptyId: tab.ptyId!, + worktreeId, + targetId, + tabId: tab.id + }) + ) + if (lease?.leafId) { + tabSnapshots.push({ leafId: lease.leafId, ptyId: tab.ptyId }) + } + } + } + } + for (const [tabId, layout] of Object.entries(session?.terminalLayoutsByTabId ?? {})) { + const worktreeId = Object.entries(session?.tabsByWorktree ?? {}).find(([, tabs]) => + tabs.some((tab) => tab.id === tabId) + )?.[0] + for (const [leafId, ptyId] of Object.entries(layout.ptyIdsByLeafId ?? {})) { + if ( + leases.some( + (lease) => + lease.leafId === leafId && + this.sshRemotePtyLeaseMayReferenceBinding(lease, { + ptyId, + targetId, + ...(worktreeId ? { worktreeId } : {}), + tabId, + leafId + }) + ) + ) { + leaves.push({ leafId, ptyId }) + } + } + } + return { tabs: tabSnapshots, leaves } + } + + private restoreSshLeaseBindings( + targetId: string, + leases: SshRemotePtyLease[], + snapshots: { + tabs: { leafId: string; ptyId: string }[] + leaves: { leafId: string; ptyId: string }[] + } + ): void { + const currentLeases = new Set(this.state.sshRemotePtyLeases ?? []) + const session = this.state.workspaceSession + const hasLiveLease = (snapshot: { leafId: string; ptyId: string }): boolean => + leases.some( + (lease) => + currentLeases.has(lease) && + lease.state !== 'terminated' && + lease.state !== 'expired' && + lease.targetId === targetId && + lease.leafId === snapshot.leafId && + lease.ptyId === this.getRelayPtyIdForSshLeaseComparison(targetId, snapshot.ptyId) + ) + for (const snapshot of snapshots.tabs) { + const tabId = findTerminalTabIdForLeaf(session, snapshot.leafId) + const tab = Object.values(session?.tabsByWorktree ?? {}) + .flat() + .find((candidate) => candidate.id === tabId) + if (hasLiveLease(snapshot) && tab?.ptyId === null) { + tab.ptyId = snapshot.ptyId + } + } + for (const snapshot of snapshots.leaves) { + const tabId = findTerminalTabIdForLeaf(session, snapshot.leafId) + const layout = tabId ? session?.terminalLayoutsByTabId?.[tabId] : undefined + if ( + hasLiveLease(snapshot) && + layout && + layout.ptyIdsByLeafId?.[snapshot.leafId] === undefined + ) { + layout.ptyIdsByLeafId = { + ...layout.ptyIdsByLeafId, + [snapshot.leafId]: snapshot.ptyId + } + } + } + } + markSshRemotePtyLeases(targetId: string, state: SshRemotePtyLease['state']): void { if (this.updateSshRemotePtyLeaseStates(targetId, state)) { this.flush() @@ -7354,6 +7687,7 @@ export class Store { state: SshRemotePtyLease['state'] ): Promise { if (this.updateSshRemotePtyLeaseStates(targetId, state)) { + this.scheduleSave() await this.flushDurableStateOrThrowAsync() } } @@ -7366,10 +7700,30 @@ export class Store { ptyIds.map((ptyId) => this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)) ) if (this.updateSshRemotePtyLeaseStates(targetId, 'attached', relayPtyIds)) { + this.scheduleSave() await this.flushDurableStateOrThrowAsync() } } + async markSshRemotePtyLeasesTerminatedAsync( + targetId: string, + ptyIds: readonly string[] + ): Promise { + const relayPtyIds = new Set( + ptyIds.map((ptyId) => this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)) + ) + if (this.updateSshRemotePtyLeaseStates(targetId, 'terminated', relayPtyIds)) { + this.scheduleSave() + try { + await this.flushDurableStateOrThrowAsync(false) + } catch (error) { + // Keep the proven exit retryable after a transient persistence failure. + this.scheduleSave() + throw error + } + } + } + private updateSshRemotePtyLeaseStates( targetId: string, state: SshRemotePtyLease['state'], @@ -7388,8 +7742,13 @@ export class Store { continue } if (state === 'detached' && lease.state !== 'attached') { + if (lease.state === 'expired') { + // A later detach fences rollback from restoring attached ownership. + this.advanceSshRemotePtyLeaseMutationVersion(lease) + } continue } + this.advanceSshRemotePtyLeaseMutationVersion(lease) if (lease.state !== state) { lease.state = state lease.updatedAt = now @@ -7418,6 +7777,7 @@ export class Store { if (!lease) { return } + this.advanceSshRemotePtyLeaseMutationVersion(lease) const shouldClearBindings = state === 'terminated' || state === 'expired' if (lease.state === state) { if (shouldClearBindings && this.clearSshRemotePtyBindingsForLeases(targetId, [lease])) { @@ -7471,19 +7831,27 @@ export class Store { this.clearSshRemotePtyBindingsForLeases(targetId, leases ?? []) } + /** `arbitratedFrom` names the plane a supersession DECISION was made from. A decision reached by + * reading one plane may only mutate that plane: the other plane's binding never got a vote, and + * deleting it strands a shell its owner can still be using. An explicit expiry or termination is + * plane-agnostic — the pty is gone for everyone — so those callers pass nothing. */ private clearSshRemotePtyBindingsForLeases( targetId: string, - leases: SshRemotePtyLease[] + leases: SshRemotePtyLease[], + arbitratedFrom?: 'local' ): boolean { if (!leases?.length) { return false } let changed = false const sessions = new Set( - [ - this.state.workspaceSession, - this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)] - ].filter((session): session is WorkspaceSessionState => Boolean(session)) + (arbitratedFrom === 'local' + ? [this.state.workspaceSession] + : [ + this.state.workspaceSession, + this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)] + ] + ).filter((session): session is WorkspaceSessionState => Boolean(session)) ) for (const session of sessions) { for (const [worktreeId, tabs] of Object.entries(session.tabsByWorktree ?? {})) { @@ -7592,10 +7960,11 @@ export class Store { // Async twin of flushOrThrow: durable state only. Active-view and GitHub sidecars are // quit/startup work and must not be snapshotted on the live SSH establish/reconnect path. - private async flushDurableStateOrThrowAsync(): Promise { + private async flushDurableStateOrThrowAsync(drainToStableGeneration = true): Promise { if (this.writesFrozen || this.quitFlushStarted) { throw new Error('Cannot flush while persistence is finalized') } + const requiredDurableGeneration = this.writeGeneration for (;;) { if (this.writeTimer) { clearTimeout(this.writeTimer) @@ -7604,6 +7973,12 @@ export class Store { this.firstPendingSaveAt = null const generation = this.writeGeneration await this.enqueueWrite() + if ( + !drainToStableGeneration && + this.lastDurableWriteGeneration >= requiredDurableGeneration + ) { + break + } if (generation === this.writeGeneration) { break } diff --git a/src/main/providers/pty-provider-contract.ts b/src/main/providers/pty-provider-contract.ts index 09f3cda62fd..5e4a34db5a4 100644 --- a/src/main/providers/pty-provider-contract.ts +++ b/src/main/providers/pty-provider-contract.ts @@ -123,7 +123,8 @@ export type IPtyProvider = { /** Whether fresh structured creates can replay one spawn across a lost relay response. */ supportsAgentSessionCreateOperations?: (options?: PtyProbeOptions) => boolean | Promise attach(id: string): Promise | void> - hasPty?: (id: string) => boolean + /** Three-valued: null means the provider cannot see this id right now — absence is unproven, never dead. */ + hasPty?: (id: string) => boolean | null /** Exact provider readback: false only when the provider answered that the PTY is absent. */ probePtyLiveness?: (id: string) => Promise write(id: string, data: string): void diff --git a/src/main/providers/ssh-pty-errors.ts b/src/main/providers/ssh-pty-errors.ts index a8c551f228b..c389422f55d 100644 --- a/src/main/providers/ssh-pty-errors.ts +++ b/src/main/providers/ssh-pty-errors.ts @@ -1,12 +1,30 @@ -export const SSH_SESSION_EXPIRED_ERROR = 'SSH_SESSION_EXPIRED' -export const SSH_PTY_IDENTITY_MISMATCH_ERROR = 'SSH_PTY_IDENTITY_MISMATCH' +export { + SSH_SESSION_EXPIRED_ERROR, + SSH_PTY_IDENTITY_MISMATCH_ERROR, + SSH_PTY_EXITED_ERROR, + SSH_SOURCE_RESTORE_REQUIRED_ERROR +} from '../../shared/ssh-pty-failure-tokens' +import { + SSH_SOURCE_RESTORE_REQUIRED_ERROR, + isSshPtyExitedMessage, + isSshPtyIdentityMismatchMessage +} from '../../shared/ssh-pty-failure-tokens' + +export function isSshSourceRestoreRequiredError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error) + return message.includes(SSH_SOURCE_RESTORE_REQUIRED_ERROR) +} export function isSshPtyNotFoundError(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error) return /PTY ".+" not found/i.test(message) } +/** The relay watched this shell exit — first-hand proof, unlike a bare not-found. */ +export function isSshPtyExitedError(error: unknown): boolean { + return isSshPtyExitedMessage(error instanceof Error ? error.message : String(error)) +} + export function isSshPtyIdentityMismatchError(error: unknown): boolean { - const message = error instanceof Error ? error.message : String(error) - return message.includes(SSH_PTY_IDENTITY_MISMATCH_ERROR) || /identity mismatch/i.test(message) + return isSshPtyIdentityMismatchMessage(error instanceof Error ? error.message : String(error)) } diff --git a/src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts b/src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts new file mode 100644 index 00000000000..86d8cc36e58 --- /dev/null +++ b/src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts @@ -0,0 +1,215 @@ +/** + * An identity mismatch means the relay FOUND the PTY and its recorded pane + * differs — the shell is running. Publishing that as expiry makes the renderer + * clear the binding and cold-restore with agent resume, which is a second agent + * on one transcript while the first keeps running. + * + * Reachable today from a shipped gesture: detaching a pane into a new tab + * changes tabId, the relay still holds the tabId frozen at spawn, and the + * reattach mismatches. + * + * Pins the PRODUCER and the CONSUMERS together. The two guards this program + * shipped before both passed while sitting off the route production takes, so + * asserting the classifier alone is not enough — the destructive token has to + * be absent from what the reattach actually throws. + */ +import { describe, expect, it, vi } from 'vitest' +import { reattachSshPtySession } from './ssh-pty-session-reattach' +import { + SSH_SESSION_EXPIRED_ERROR, + SSH_PTY_IDENTITY_MISMATCH_ERROR, + formatPtyExitedError +} from '../../shared/ssh-pty-failure-tokens' + +/** The renderer converts a failure into `sessionExpired: true` on these tokens + * and then respawns on the flag alone, never consulting the classifier. + * Mirrors pty-transport.ts. */ +const SSH_PTY_CONNECTION_MISMATCH_MARKER = 'belongs to SSH connection' +function transportWouldReportSessionExpired(message: string): boolean { + return ( + message.includes(SSH_SESSION_EXPIRED_ERROR) || + message.includes(SSH_PTY_CONNECTION_MISMATCH_MARKER) + ) +} + +const CONNECTION_ID = 'ssh-target-1' +const RELAY_PTY_ID = 'pty-7' + +function muxThatFailsAttachWith(message: string): { mux: unknown } { + return { + mux: { + request: vi.fn().mockRejectedValue(new Error(message)), + notify: vi.fn() + } + } +} + +async function reattachError( + attachFailure: string, + expectedIncarnationId?: string +): Promise { + const { mux } = muxThatFailsAttachWith(attachFailure) + try { + await reattachSshPtySession({ + mux: mux as never, + connectionId: CONNECTION_ID, + sessionId: RELAY_PTY_ID, + options: { + cols: 80, + rows: 24, + paneKey: 'tab-new:leaf-1', + tabId: 'tab-new', + ...(expectedIncarnationId ? { expectedIncarnationId } : {}) + } as never + }) + } catch (error) { + return error as Error + } + throw new Error('reattach unexpectedly succeeded') +} + +describe('an identity mismatch is not a death', () => { + it('does not publish the destructive expiry token', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + }) + + // The renderer keys off this token; its own clauses live beside the + // classifier, in reattach-failure-classification.test.ts. + it('marks the failure with the shared mismatch token', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`) + + expect(error.message).toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR) + }) + + // The flag path bypasses the classifier entirely, so it needs its own clause. + it('does not trip the renderer transport into sessionExpired', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`) + + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) +}) + +describe('only an exit the relay watched is a death', () => { + // Clause-selectivity: a fix that silences real expiry would strand panes on a + // shell that truly went away, so the contrast has to hold. + it('publishes expiry when the relay reports the exit it observed', async () => { + const error = await reattachError( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-host-7'), + 'inc-host-7' + ) + + expect(error.message).toContain(SSH_SESSION_EXPIRED_ERROR) + expect(error.message).not.toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(true) + }) + + // INVERTED. This used to require a bare absence to publish expiry. That is the defect: the relay + // we asked cannot hand the id back, which proves an exit ONLY if it is the relay that minted it. + // A replaced relay answers exactly this for shells still running under its predecessor, so + // expiry there cleared ownership and resumed the agent a second time onto a live shell. The + // death that is real is still detected — by the clause above, from an exit the relay watched. + // Enforcement cannot live only on the host: the host is the party whose answer is in question, + // and versions differ. A proof we cannot tie to the shell we asked about is not proof. + it('does not publish expiry for an exit that names a different shell', async () => { + const error = await reattachError( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-some-other-shell'), + 'inc-host-7' + ) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) + + it('does not publish expiry for an exit that names a different PTY id', async () => { + const error = await reattachError( + formatPtyExitedError('pty-some-other-shell', 0, 'inc-host-7'), + 'inc-host-7' + ) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) + + it('does not publish expiry for an exit when the pane knows no incarnation', async () => { + const error = await reattachError(formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-host-7')) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + }) + + it('does not publish expiry when the relay merely has no such PTY', async () => { + const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found`) + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(transportWouldReportSessionExpired(error.message)).toBe(false) + }) + + it('leaves an unrelated failure untouched rather than guessing', async () => { + const error = await reattachError('ECONNRESET while writing to the relay') + + expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR) + expect(error.message).not.toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR) + }) +}) + +// New relays prioritize incarnation, so the pane fields do not reject a moved pane. Old relays +// ignore the additive incarnation field and still need their weaker pane fence. +describe('reattach preserves the old-relay pane fence', () => { + async function attachParams( + extraOptions: Record = {} + ): Promise> { + const request = vi.fn().mockRejectedValue(new Error('boom')) + try { + await reattachSshPtySession({ + mux: { request, notify: vi.fn() } as never, + connectionId: CONNECTION_ID, + sessionId: RELAY_PTY_ID, + options: { + cols: 80, + rows: 24, + paneKey: 'tab-new:leaf-1', + tabId: 'tab-new', + env: { ORCA_PANE_KEY: 'tab-old:leaf-1', ORCA_TAB_ID: 'tab-old' }, + ...extraOptions + } as never + }) + } catch { + // the attach failure is not what this clause is about + } + const call = request.mock.calls.at(0) + return (call?.[1] ?? call?.[0] ?? {}) as Record + } + + it('sends the current pane key for relays that do not understand incarnation', async () => { + expect(await attachParams()).toMatchObject({ expectedPaneKey: 'tab-new:leaf-1' }) + }) + + it('sends the current tab id for relays that do not understand incarnation', async () => { + expect(await attachParams()).toMatchObject({ expectedTabId: 'tab-new' }) + }) + + // The producer pin for the relay's incarnation guard. The guard is only worth having if the + // expectation actually leaves the client, and nothing else here would notice if it stopped: + // the relay stays permissive on an absent field, so a silent regression reads as "all green". + it('sends the expected incarnation when the host attested one', async () => { + expect(await attachParams({ expectedIncarnationId: 'inc-host-7' })).toMatchObject({ + expectedIncarnationId: 'inc-host-7', + expectedPaneKey: 'tab-new:leaf-1', + expectedTabId: 'tab-new' + }) + }) + + // A stand-in minted locally when the host reported none. It is first-write-wins and is dropped + // when provider state resets, so the same live shell can present a different one after a + // reconnect — sending it would make the relay refuse the pane its own shell. + it('sends no expected incarnation when the value was synthesized locally', async () => { + expect(await attachParams({ expectedIncarnationId: 'legacy:23:0:pty-7' })).not.toHaveProperty( + 'expectedIncarnationId' + ) + }) + + it('sends no expected incarnation when there is none to send', async () => { + expect(await attachParams()).not.toHaveProperty('expectedIncarnationId') + }) +}) diff --git a/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts b/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts index e088b21f025..b150c7fa0c7 100644 --- a/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts +++ b/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts @@ -316,7 +316,8 @@ describe('SSH fresh agent-session create operations', () => { ) expect(onData).not.toHaveBeenCalled() - expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(false) + // Not `false`: the provider has never listed this host, so absence is unknown, not proven. + expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).not.toBe(true) const shutdownRequest = await waitForRequest(transport, 'pty.shutdown') transport.deliver(responseFrame(shutdownRequest.id as number, null, 4)) const cancelRequest = await waitForRequest(transport, 'pty.cancelDelivery') diff --git a/src/main/providers/ssh-pty-provider-exit-race.test.ts b/src/main/providers/ssh-pty-provider-exit-race.test.ts index 22a19985d21..38c665d93c4 100644 --- a/src/main/providers/ssh-pty-provider-exit-race.test.ts +++ b/src/main/providers/ssh-pty-provider-exit-race.test.ts @@ -170,3 +170,30 @@ it('returns a provisional source activation lease to reconnect authority', async deliveryToken: 'token-reconnect' }) }) + +it('fences a mismatched attach response before installing its source activation', async () => { + const response = { + incarnationId: 'incarnation-other', + sourceActivation: sourceActivation('incarnation-other') + } + const mux = { + request: vi.fn(async (method: string, _params, options) => { + if (method !== 'pty.attach') { + return undefined + } + options?.beforeResolve?.(response) + return response + }), + notify: vi.fn(), + onNotification: vi.fn(), + dispose: vi.fn(), + isDisposed: vi.fn().mockReturnValue(false) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.attachForReconnect('ssh:conn-1@@pty-1', undefined, 'incarnation-expected') + ).rejects.toThrow('SSH_PTY_IDENTITY_MISMATCH') + + expect(mux.request).toHaveBeenCalledTimes(1) +}) diff --git a/src/main/providers/ssh-pty-provider-output-state.ts b/src/main/providers/ssh-pty-provider-output-state.ts index 4d98097b9c6..a80ffc4ec4f 100644 --- a/src/main/providers/ssh-pty-provider-output-state.ts +++ b/src/main/providers/ssh-pty-provider-output-state.ts @@ -11,6 +11,7 @@ import { type SshPtyReceivingActivationLease } from './ssh-pty-notification-routing' import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' +import { SYNTHESIZED_PTY_INCARNATION_PREFIX } from '../../shared/pty-incarnation' export class SshPtyProviderOutputState { private readonly dataListeners = new Set() @@ -132,7 +133,7 @@ export class SshPtyProviderOutputState { this.rememberPtyIncarnation(relayPtyId, incarnationId) let resolved = this.incarnationByRelayPtyId.get(relayPtyId) if (!resolved) { - resolved = `legacy:${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}` + resolved = `${SYNTHESIZED_PTY_INCARNATION_PREFIX}${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}` this.incarnationByRelayPtyId.set(relayPtyId, resolved) } return resolved diff --git a/src/main/providers/ssh-pty-provider-process-events.test.ts b/src/main/providers/ssh-pty-provider-process-events.test.ts index e152ef47fae..449c3e4c356 100644 --- a/src/main/providers/ssh-pty-provider-process-events.test.ts +++ b/src/main/providers/ssh-pty-provider-process-events.test.ts @@ -29,6 +29,17 @@ describe('SshPtyProvider process listings and events', () => { provider = new SshPtyProvider('conn-1', mux as never) }) + // Why: a reconnect builds a fresh provider with an empty set, so a miss before the + // first completed listing is ignorance about the host, not a dead PTY. + it('answers unknown liveness until a completed listing makes absence provable', async () => { + expect(provider.hasPty(scopedPty1)).toBe(null) + + mux.request.mockResolvedValueOnce([]) + await provider.listProcesses() + + expect(provider.hasPty(scopedPty1)).toBe(false) + }) + it('scopes process listings and bounds the relay RPC by the teardown deadline', async () => { const processes = [{ id: 'pty-1', cwd: '/home', title: 'zsh', worktreeId: 'repo::/home' }] mux.request.mockResolvedValue(processes) diff --git a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts index 74989dc34e5..af7eeb98139 100644 --- a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts +++ b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts @@ -1,8 +1,28 @@ import { describe, expect, it, vi } from 'vitest' -import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' +import { + SSH_PTY_IDENTITY_MISMATCH_ERROR, + SSH_SESSION_EXPIRED_ERROR, + SSH_SOURCE_RESTORE_REQUIRED_ERROR +} from './ssh-pty-errors' import { SshPtyProvider } from './ssh-pty-provider' describe('SSH PTY provider session reattach incarnation', () => { + it.each([ + ['an old-relay response without an incarnation', {}], + ['a response for a different incarnation', { incarnationId: 'incarnation-other' }] + ])('rejects %s', async (_label, response) => { + const mux = { + request: vi.fn().mockResolvedValue(response), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.attachForReconnect('ssh:conn-1@@pty-old', undefined, 'incarnation-expected') + ).rejects.toThrow(SSH_PTY_IDENTITY_MISMATCH_ERROR) + }) + it('remembers the authoritative incarnation before a legacy exit arrives', async () => { let notify: ((method: string, params: Record) => void) | undefined const mux = { @@ -44,8 +64,10 @@ describe('SSH PTY provider session reattach incarnation', () => { } const provider = new SshPtyProvider('conn-1', mux as never) - await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( - `${SSH_SESSION_EXPIRED_ERROR}: pty-old` - ) + // Fails closed, but NOT as expiry: the shell is still running, and callers + // respawn on expiry — which duplicate-resumed the live agent session. + const spawn = provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) + await expect(spawn).rejects.toThrow(`${SSH_SOURCE_RESTORE_REQUIRED_ERROR}: pty-old`) + await expect(spawn).rejects.not.toThrow(SSH_SESSION_EXPIRED_ERROR) }) }) diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 40301906f52..e0b1d6b8213 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' +import { formatPtyExitedError } from '../../shared/ssh-pty-failure-tokens' import { SshPtyProvider } from './ssh-pty-provider' import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env' import { PTY_STARTUP_INGRESS_VERSION } from '../../shared/pty-startup-ingress' @@ -584,7 +585,8 @@ describe('SshPtyProvider', () => { id: 'pty-old', cols: 80, rows: 24, - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true }, sourceActivationRequestOptions ) @@ -630,7 +632,8 @@ describe('SshPtyProvider', () => { id: 'pty-old', cols: 80, rows: 24, - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true }) expect(result).toEqual({ id: 'ssh:conn-1@@pty-old', @@ -639,7 +642,8 @@ describe('SshPtyProvider', () => { }) }) - it('reattaches with explicit pane identity when hook env was stripped', async () => { + // New relays prioritize incarnation; old relays ignore that field and retain this weaker fence. + it('preserves pane identity for relays that predate incarnation fencing', async () => { mux.request.mockResolvedValue({ replay: 'buffered-output' }) await provider.spawn({ @@ -655,18 +659,30 @@ describe('SshPtyProvider', () => { cols: 80, rows: 24, suppressReplayNotification: true, + exitProofSupported: true, expectedPaneKey: 'tab-a:leaf-a', expectedTabId: 'tab-a' }) }) - it('does not fresh-spawn over an expired reattach session', async () => { - mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found')) - - await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( - 'SSH_SESSION_EXPIRED: pty-old' + it('does not fresh-spawn over a reattach whose shell the relay saw exit', async () => { + // INVERTED to drive the exit the relay OBSERVED. A bare not-found no longer becomes expiry — + // it proves nothing when the answering relay may be a replacement — so the observed exit is + // now what carries this through to expiry. The property under test is unchanged: a failed + // reattach throws instead of quietly spawning a second shell over the pane. + mux.request.mockRejectedValueOnce( + new Error(formatPtyExitedError('pty-old', 0, 'inc-host-old')) ) + await expect( + provider.spawn({ + cols: 80, + rows: 24, + sessionId: 'pty-old', + expectedIncarnationId: 'inc-host-old' + }) + ).rejects.toThrow('SSH_SESSION_EXPIRED: pty-old') + expect(mux.request).toHaveBeenNthCalledWith( 1, 'pty.attach', @@ -674,13 +690,27 @@ describe('SshPtyProvider', () => { id: 'pty-old', cols: 80, rows: 24, - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: 'inc-host-old' }, sourceActivationRequestOptions ) expect(mux.request).toHaveBeenCalledTimes(1) }) + // The sibling case, and the one the change is for: an id the relay simply does not know may be + // a shell still running under a relay this one replaced. It must still refuse to fresh-spawn, + // and must NOT be dressed up as expiry, which is what authorized the replacement. + it('does not fresh-spawn, nor claim expiry, when the relay merely has no such PTY', async () => { + mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found')) + + await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( + 'PTY "pty-old" not found' + ) + expect(mux.request).toHaveBeenCalledTimes(1) + }) + it('preserves transient reattach failures for retry handling', async () => { mux.request.mockRejectedValueOnce(new Error('SSH connection lost, reconnecting...')) @@ -714,7 +744,8 @@ describe('SshPtyProvider', () => { 'pty.attach', { id: 'pty-1', - suppressReplayNotification: true + suppressReplayNotification: true, + exitProofSupported: true }, expect.objectContaining({ timeoutMs: 10_000, @@ -739,28 +770,6 @@ describe('SshPtyProvider', () => { ) }) - it('attachForReconnect forwards expected identity when provided', async () => { - await provider.attachForReconnect(scopedPty1, { - paneKey: 'tab-a:leaf-a', - tabId: 'tab-a' - }) - - expectRequest( - mux.request, - 'pty.attach', - { - id: 'pty-1', - suppressReplayNotification: true, - expectedPaneKey: 'tab-a:leaf-a', - expectedTabId: 'tab-a' - }, - expect.objectContaining({ - timeoutMs: 10_000, - beforeResolve: expect.any(Function) - }) - ) - }) - it('write sends pty.data notification', () => { provider.write(scopedPty1, 'hello') expect(mux.notify).toHaveBeenCalledWith('pty.data', { id: 'pty-1', data: 'hello' }) diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 2bc52fe491d..2eaf202f0b2 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -13,6 +13,7 @@ import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state' import { spawnFreshSshPty } from './ssh-agent-session-create-operation' import { mapSshPtyProcessList } from './ssh-agent-session-process-list' import { + buildSshPtyReconnectAttachParams, requestSshPtyAttach, reattachSshPtySessionWithExitFence, type PtySourceRecoveryRequest, @@ -22,7 +23,7 @@ import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request' import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities' import type { PtyProcessInspection } from './pty-process-inspection' -import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' +import { SSH_SOURCE_RESTORE_REQUIRED_ERROR } from './ssh-pty-errors' // Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch. function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined { @@ -34,6 +35,7 @@ export class SshPtyProvider implements IPtyProvider { private mux: SshChannelMultiplexer private connectionId: string private livePtyIds = new Set() + private listedOnce = false readonly getAppliedSize: NonNullable private readonly agentSessionCapabilities: SshAgentSessionCapabilities private spawnExitRaces = new SshPtySpawnExitRaceTracker() @@ -63,6 +65,7 @@ export class SshPtyProvider implements IPtyProvider { dispose(): void { this.outputState.dispose() this.livePtyIds.clear() + this.listedOnce = false } getConnectionId = (): string => this.connectionId @@ -101,8 +104,11 @@ export class SshPtyProvider implements IPtyProvider { this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId) }) if (result.sourceRecovery?.status === 'restoreRequired') { + // Why not SSH_SESSION_EXPIRED: the shell is still running, only its + // output source needs re-establishing. Reporting expiry made the pane + // respawn and resume the same agent session twice into one transcript. throw new Error( - `${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}` + `${SSH_SOURCE_RESTORE_REQUIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}` ) } this.livePtyIds.add(result.id) @@ -177,20 +183,16 @@ export class SshPtyProvider implements IPtyProvider { async attachForReconnect( id: string, - expected?: { paneKey?: string; tabId?: string }, - sourceRecovery?: PtySourceRecoveryRequest + sourceRecovery?: PtySourceRecoveryRequest, + expectedIncarnationId?: string, + legacyExpectedIdentity?: { paneKey?: string; tabId?: string } ): Promise { - // Why: reconnect owns replay delivery so stale/duplicate attach results can - // be filtered before they reach the renderer. The expected identity lets the - // relay reject a cross-generation id collision instead of reattaching this - // lease to a different pane's freshly spawned PTY. - const params = { + const params = buildSshPtyReconnectAttachParams({ id: this.toRelayPtyId(id), - suppressReplayNotification: true, ...(sourceRecovery ? { sourceRecovery } : {}), - ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), - ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) - } + ...(expectedIncarnationId ? { expectedIncarnationId } : {}), + ...(legacyExpectedIdentity ? { legacyExpectedIdentity } : {}) + }) const relayPtyId = this.toRelayPtyId(id) return await requestSshPtyAttach({ mux: this.mux, @@ -296,11 +298,14 @@ export class SshPtyProvider implements IPtyProvider { const relayPtyId = this.toRelayPtyId(process.id) this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId) } + this.listedOnce = true return processes } - hasPty(id: string): boolean { - return this.livePtyIds.has(id) + hasPty(id: string): boolean | null { + // Why null before a completed listing: a reconnect builds a new provider with an + // empty set, so a miss there is ignorance about the host, not a dead PTY. + return this.livePtyIds.has(id) ? true : this.listedOnce ? false : null } async getDefaultShell(): Promise { diff --git a/src/main/providers/ssh-pty-session-reattach.ts b/src/main/providers/ssh-pty-session-reattach.ts index 34974940caf..ff490464853 100644 --- a/src/main/providers/ssh-pty-session-reattach.ts +++ b/src/main/providers/ssh-pty-session-reattach.ts @@ -1,11 +1,16 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' -import { isPtyIncarnationId, type PtyIncarnationId } from '../../shared/pty-incarnation' +import { + isPtyIncarnationId, + isRelayAttestedPtyIncarnationId, + type PtyIncarnationId +} from '../../shared/pty-incarnation' import { SSH_PTY_IDENTITY_MISMATCH_ERROR, SSH_SESSION_EXPIRED_ERROR, - isSshPtyIdentityMismatchError, - isSshPtyNotFoundError + isSshPtyExitedError, + isSshPtyIdentityMismatchError } from './ssh-pty-errors' +import { parseMatchingPtyExitedError } from '../../shared/ssh-pty-failure-tokens' import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' import type { PtySpawnOptions, PtySpawnResult } from './types' import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' @@ -27,6 +32,29 @@ export type SshPtyAttachResult = { sourceActivationLease?: SshPtyReceivingActivationLease } +export function buildSshPtyReconnectAttachParams(args: { + id: string + sourceRecovery?: PtySourceRecoveryRequest + expectedIncarnationId?: string + legacyExpectedIdentity?: { paneKey?: string; tabId?: string } +}): Record { + return { + id: args.id, + suppressReplayNotification: true, + exitProofSupported: true, + ...(isRelayAttestedPtyIncarnationId(args.expectedIncarnationId) + ? { expectedIncarnationId: args.expectedIncarnationId } + : {}), + ...(args.legacyExpectedIdentity?.paneKey + ? { expectedPaneKey: args.legacyExpectedIdentity.paneKey } + : {}), + ...(args.legacyExpectedIdentity?.tabId + ? { expectedTabId: args.legacyExpectedIdentity.tabId } + : {}), + ...(args.sourceRecovery ? { sourceRecovery: args.sourceRecovery } : {}) + } +} + type SshPtyReattachResult = PtySpawnResult & { sourceRecovery?: PtySourceRecoveryResult sourceActivationLease?: SshPtyReceivingActivationLease @@ -85,6 +113,18 @@ export async function requestSshPtyAttach(args: { rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void }): Promise { let activationLease: SshPtyReceivingActivationLease | undefined + const parseResult = (value: unknown): SshPtyAttachResult => { + const result = parseSshPtyAttachResult(value) + const expectedIncarnationId = args.params.expectedIncarnationId + // Old relays ignore the request fence, so their response must prove the same shell. + if ( + isRelayAttestedPtyIncarnationId(expectedIncarnationId) && + result.incarnationId !== expectedIncarnationId + ) { + throw new Error(`${SSH_PTY_IDENTITY_MISMATCH_ERROR}: ${args.relayPtyId}`) + } + return result + } const installFromResult = (result: SshPtyAttachResult): void => { if (!activationLease && result.sourceActivation && args.installSourceActivation) { activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation) @@ -93,9 +133,9 @@ export async function requestSshPtyAttach(args: { try { const rawResult = await args.mux.request('pty.attach', args.params, { ...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }), - beforeResolve: (value) => installFromResult(parseSshPtyAttachResult(value)) + beforeResolve: (value) => installFromResult(parseResult(value)) }) - const result = parseSshPtyAttachResult(rawResult) + const result = parseResult(rawResult) installFromResult(result) args.rememberPtyIncarnation?.(args.relayPtyId, result.incarnationId) if (args.commitSourceActivation) { @@ -185,7 +225,6 @@ export async function reattachSshPtySession(args: { const relaySessionId = toRelaySshPtyId(args.connectionId, args.sessionId) console.warn(`[ssh-pty] spawn() called with sessionId=${args.sessionId}, attempting pty.attach`) try { - // Why: expected pane identity prevents a reused relay id from attaching the wrong shell. const expectedPaneKey = args.options.paneKey ?? args.options.env?.ORCA_PANE_KEY const expectedTabId = args.options.tabId ?? args.options.env?.ORCA_TAB_ID const attachResult = await requestSshPtyAttach({ @@ -196,6 +235,17 @@ export async function reattachSshPtySession(args: { cols: args.options.cols, rows: args.options.rows, suppressReplayNotification: true, + // Declares that this client can act on a proven exit. Without it a host must keep answering + // in the older wording, because an older client reads anything else as an unknown failure + // and leaves the pane with no shell and no way back. + exitProofSupported: true, + // The shell's own identity, so it survives a pane moving between tabs — unlike the pane + // identity this replaced. Sent only when the host attested it: a locally synthesized + // stand-in is not stable across reconnects and would refuse the pane its own shell. An + // older relay ignores the field, so the legacy pane fence below remains its fallback. + ...(isRelayAttestedPtyIncarnationId(args.options.expectedIncarnationId) + ? { expectedIncarnationId: args.options.expectedIncarnationId } + : {}), ...(expectedPaneKey ? { expectedPaneKey } : {}), ...(expectedTabId ? { expectedTabId } : {}) }, @@ -219,12 +269,34 @@ export async function reattachSshPtySession(args: { } catch (error) { // Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding. console.warn(`[ssh-pty] pty.attach FAILED for ${args.sessionId}:`, error) - if (isSshPtyNotFoundError(error)) { - const mismatchMarker = isSshPtyIdentityMismatchError(error) - ? ` ${SSH_PTY_IDENTITY_MISMATCH_ERROR}` - : '' - throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}${mismatchMarker}`) + // Why: the relay reports a mismatch by saying "not found", but it found the + // pty — comparing identity is how it knows. Publishing expiry there makes + // the renderer respawn and resume the agent a second time onto a live shell. + if (isSshPtyIdentityMismatchError(error)) { + throw new Error(`${SSH_PTY_IDENTITY_MISMATCH_ERROR}: ${relaySessionId}`) } + // The relay WATCHED this shell exit, which is the only answer that proves it is gone, so this + // is the one route that may authorize a replacement. The proof still has to be about OUR shell: + // the host applies that rule too, but the host is the party whose answer is in question and + // versions differ, so a proof we cannot tie to the incarnation we asked about is not proof and + // falls through to the disconnected pane instead of replacing a shell that may be running. + if (isSshPtyExitedError(error)) { + const proof = parseMatchingPtyExitedError( + error instanceof Error ? error.message : String(error), + relaySessionId, + args.options.expectedIncarnationId + ) + if (proof) { + throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}`) + } + throw error + } + // A bare not-found deliberately does NOT become expiry any more. It means the relay we asked + // cannot hand the id back, which is proof of an exit only if that relay is the one that minted + // it — and a replaced relay answers exactly this for shells still running under its + // predecessor. Treating it as death cleared ownership and resumed the agent a second time onto + // a live shell. Unproven now falls through to the caller, which shows the pane as + // disconnected and lets the user decide. throw error } } diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index aa68dad27ae..9d141068caa 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -2848,7 +2848,9 @@ describe('OrcaRuntimeService', () => { ) }) - it('recovers a disconnected pane through one HUB-owned replacement', async () => { + // Inverted for STA-3077 S8: the replacement grant this pinned was unreachable in production + // (relay-native lease ptyId vs app-form runtime ptyId) and has been deleted. + it('refuses to replace a disconnected SSH pane, even inside the lease grace window', async () => { const tabId = 'tab-recover' const runtime = createRuntimeWithSshLease('pty-expired', tabId) const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) @@ -2870,18 +2872,8 @@ describe('OrcaRuntimeService', () => { await expect( runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) - ).resolves.toMatchObject({ - handle: 'term-replacement', - tabId, - leafId: HEADLESS_LEAF_ID, - worktreeId: TEST_WORKTREE_ID - }) - expect(createTerminal).toHaveBeenCalledWith(`id:${TEST_WORKTREE_ID}`, { - tabId, - leafId: HEADLESS_LEAF_ID, - focus: false, - persistHostSessionBinding: true - }) + ).rejects.toThrow('terminal_not_recoverable') + expect(createTerminal).not.toHaveBeenCalled() }) it('rejects missing host panes without authoritative expired binding evidence', async () => { @@ -2947,7 +2939,9 @@ describe('OrcaRuntimeService', () => { expect(createTerminal).not.toHaveBeenCalled() }) - it('deduplicates concurrent pane recovery across stale viewer handles', async () => { + // Inverted for STA-3077 S8: this was the only consumer of the recovery-dedup map, which existed + // solely to de-duplicate the (unreachable, now deleted) replacement grant. Scenario kept. + it('concurrent recovery requests for one disconnected pane both refuse and create nothing', async () => { const tabId = 'tab-concurrent' const runtime = createRuntimeWithSshLease('pty-expired', tabId) const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) @@ -2957,30 +2951,23 @@ describe('OrcaRuntimeService', () => { }) const expiredHandle = runtime.resolveTerminalPane(paneKey, TEST_WORKTREE_ID).handle runtime.onPtyExit('pty-expired', 0) - let finishCreate!: (result: RuntimeTerminalCreate) => void - const pendingCreate = new Promise((resolve) => { - finishCreate = resolve - }) - const createTerminal = vi.spyOn(runtime, 'createTerminal').mockReturnValue(pendingCreate) + // Never settles: a resurrected grant would hang the assertion rather than quietly pass. + const createTerminal = vi + .spyOn(runtime, 'createTerminal') + .mockReturnValue(new Promise(() => {})) const first = runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) const second = runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, 'term-other-viewer') - finishCreate({ - handle: 'term-replacement', - tabId, - paneKey, - ptyId: 'pty-replacement', - worktreeId: TEST_WORKTREE_ID, - title: null, - surface: 'background' - }) - await expect(first).resolves.toEqual(expect.objectContaining({ handle: 'term-replacement' })) + await expect(first).rejects.toThrow('terminal_not_recoverable') + // The second viewer's handle never mapped to this pane, so it still fails the identity guard. await expect(second).rejects.toThrow('terminal_not_found') - expect(createTerminal).toHaveBeenCalledOnce() + expect(createTerminal).not.toHaveBeenCalled() }) - it('clears a failed pane recovery so a later reconnect can retry', async () => { + // Inverted for STA-3077 S8: it pinned retry-after-a-failed-grant, and the grant (unreachable at + // the pty-id shapes production mints) is deleted, so there is nothing left to retry into. + it('a repeated recovery request never accumulates a grant', async () => { const tabId = 'tab-retry' const runtime = createRuntimeWithSshLease('pty-expired', tabId) const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) @@ -2990,6 +2977,7 @@ describe('OrcaRuntimeService', () => { }) const expiredHandle = runtime.resolveTerminalPane(paneKey, TEST_WORKTREE_ID).handle runtime.onPtyExit('pty-expired', 0) + // A fully working createTerminal stays armed across both calls to prove neither one reaches it. const createTerminal = vi .spyOn(runtime, 'createTerminal') .mockRejectedValueOnce(new Error('relay_reconnecting')) @@ -3005,13 +2993,16 @@ describe('OrcaRuntimeService', () => { await expect( runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) - ).rejects.toThrow('relay_reconnecting') + ).rejects.toThrow('terminal_not_recoverable') await expect( runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle) - ).resolves.toMatchObject({ handle: 'term-retry' }) - expect(createTerminal).toHaveBeenCalledTimes(2) + ).rejects.toThrow('terminal_not_recoverable') + expect(createTerminal).toHaveBeenCalledTimes(0) }) + // STA-3077 S8: no longer discriminating — since the grant was deleted, an 'expired' lease refuses + // identically (see the grace-window case above), so 'terminated' no longer selects the outcome. + // Retained only as a regression guard: if a replacement path ever returns, this must stay a refusal. it('does not recover a pane whose authoritative SSH lease was terminated', async () => { const tabId = 'tab-terminated' const runtime = createRuntimeWithSshLease('pty-terminated', tabId, 'terminated') diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 55d2195c31e..856b5960b32 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2771,7 +2771,6 @@ export class OrcaRuntimeService { } >() private terminalSleepGeneration = 0 - private terminalPaneRecoveryByIdentity = new Map>() // Why: idempotency map for worktree.create — a create interrupted by a mobile // connection migration is retried with the same clientMutationId and returns // the in-flight (or just-finished) operation instead of a duplicate worktree. @@ -6211,8 +6210,7 @@ export class OrcaRuntimeService { private getRecentExpiredSshLease( worktreeId: string, tabId: string, - leafId: string | undefined, - ptyId?: string + leafId: string | undefined ): ReturnType>[number] | null { const now = Date.now() return ( @@ -6223,7 +6221,6 @@ export class OrcaRuntimeService { lease.state === 'expired' && lease.worktreeId === worktreeId && lease.tabId === tabId && - (ptyId === undefined || lease.ptyId === ptyId) && (leafId === undefined || lease.leafId === undefined || lease.leafId === leafId) && lease.updatedAt <= now && now - lease.updatedAt <= SSH_PANE_RECOVERY_GRACE_MS @@ -16525,46 +16522,17 @@ export class OrcaRuntimeService { ) { throw new Error('terminal_not_found') } - const recoveryKey = `${expectedWorktreeId}\0${paneKey}` - const pending = this.terminalPaneRecoveryByIdentity.get(recoveryKey) - if (pending) { - return pending - } if (pty?.connected) { const current = this.resolveTerminalPane(paneKey, expectedWorktreeId) if (expectedHandle === undefined || current.handle !== expectedHandle) { return current } - throw new Error('terminal_not_recoverable') } - if ( - !this.getRecentExpiredSshLease(expectedWorktreeId, parsed.tabId, parsed.leafId, pty.ptyId) - ) { - // Why: an explicit close leaves a terminated lease; only relay expiry authorizes shell recreation. - throw new Error('terminal_not_recoverable') - } - // Why: disconnected PTYs can reissue handles during graph cleanup; only a connected replacement satisfies the pane CAS. - const recovery = this.createTerminal(`id:${expectedWorktreeId}`, { - tabId: parsed.tabId, - leafId: parsed.leafId, - focus: false, - // Why: the HUB renderer may publish its exited layout while recovery is in flight; persist the replacement before that stale graph can orphan it. - persistHostSessionBinding: true - }).then((terminal) => ({ - handle: terminal.handle, - tabId: parsed.tabId, - leafId: parsed.leafId, - ptyId: terminal.ptyId ?? null, - worktreeId: expectedWorktreeId - })) - this.terminalPaneRecoveryByIdentity.set(recoveryKey, recovery) - const clearRecovery = (): void => { - if (this.terminalPaneRecoveryByIdentity.get(recoveryKey) === recovery) { - this.terminalPaneRecoveryByIdentity.delete(recoveryKey) - } - } - void recovery.then(clearRecovery, clearRecovery) - return recovery + // A disconnected pane is never silently replaced. The grant this used to hold could not fire: + // it compared a relay-native lease id against an app-form runtime id (STA-3077 S6/S8, oracle in + // ssh-pane-recovery-grant-reachability.test.ts). The pane surfaces as disconnected instead, and + // spawning a replacement is the user's explicit choice. + throw new Error('terminal_not_recoverable') } async showTerminal(handle: string): Promise { @@ -30128,13 +30096,20 @@ export class OrcaRuntimeService { // The sync hasPty rescue closes the spawn/list race: a just-spawned PTY can // register after the inventory snapshot, and federation reads one // connected:false as exited. + let directLiveness: boolean | null | undefined + try { + directLiveness = leaf.ptyId ? this.ptyController?.hasPty?.(leaf.ptyId) : undefined + } catch { + directLiveness = null + } const provenAbsent = provenLivePtyIds !== null && leaf.ptyId !== null && !provenLivePtyIds.has(leaf.ptyId) && !leaf.ptyId.startsWith('remote:') && parseAppSshPtyId(leaf.ptyId) === null && - this.ptyController?.hasPty?.(leaf.ptyId) !== true + directLiveness !== true && + directLiveness !== null return { handle: this.issueHandle(leaf), ptyId: leaf.ptyId, @@ -37335,16 +37310,14 @@ function runtimePathsEqual(left: string, right: string): boolean { * Windows/WSL/SSH ids still match themselves across hosts. */ function runtimeWorktreeIdsEqual(left: string, right: string): boolean { - const parsedLeft = splitWorktreeId(left) - const parsedRight = splitWorktreeId(right) - return parsedLeft && parsedRight - ? parsedLeft.repoId === parsedRight.repoId && - runtimePathsEqual(parsedLeft.worktreePath, parsedRight.worktreePath) - : left === right + // Why: derived from the key rather than re-parsed, so equality and the sleep / + // mutation-queue keying can never drift apart into two different identity rules. + return runtimeWorktreeIdentityKey(left) === runtimeWorktreeIdentityKey(right) } function runtimeWorktreeIdentityKey(worktreeId: string): string { // Same suffix rule: this keys PTY refresh, sleep, and mutation-queue state per session. + // NUL cannot occur in a repoId or a path, so the joined key is unambiguous. const parsed = splitWorktreeId(worktreeId) return parsed ? `${parsed.repoId}\0${normalizeRuntimePathForComparison(parsed.worktreePath)}` diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index ec279aef763..a1ab618847c 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -39,7 +39,7 @@ import type { } from './types' import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display' import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-contract' -import { parsePaneKey } from '../../../shared/stable-pane-id' +import { isEquivalentPaneKey } from '../../../shared/stable-pane-id' import { OrchestrationError } from './orchestration-error' import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew' import { @@ -63,16 +63,6 @@ import { migrateMutationReceiptCapacity } from './mutation-receipt-capacity' -// Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. -function isEquivalentPaneKey(a: string, b: string): boolean { - if (a === b) { - return true - } - const aLeaf = parsePaneKey(a)?.leafId - const bLeaf = parsePaneKey(b)?.leafId - return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) -} - function parseWorkerTerminalPriorOwnerIds(value: string): string[] | null { try { const parsed: unknown = JSON.parse(value) diff --git a/src/main/runtime/orchestration/lifecycle-reconciliation.ts b/src/main/runtime/orchestration/lifecycle-reconciliation.ts index 3d2793413b7..7e9c2bc0efa 100644 --- a/src/main/runtime/orchestration/lifecycle-reconciliation.ts +++ b/src/main/runtime/orchestration/lifecycle-reconciliation.ts @@ -1,17 +1,6 @@ import type { OrchestrationDb } from './db' import type { MessageRow, WorkerReportOutcome } from './types' -import { parsePaneKey } from '../../../shared/stable-pane-id' - -// Why: the tab half can change on pane break-out, while opaque legacy keys -// have no safe equivalence beyond exact equality. -function isSamePane(assigneePaneKey: string, senderPaneKey: string): boolean { - if (assigneePaneKey === senderPaneKey) { - return true - } - const assigneeLeaf = parsePaneKey(assigneePaneKey)?.leafId - const senderLeaf = parsePaneKey(senderPaneKey)?.leafId - return Boolean(assigneeLeaf && senderLeaf && assigneeLeaf === senderLeaf) -} +import { isEquivalentPaneKey } from '../../../shared/stable-pane-id' function hasLifecycleAuthority( dispatch: { assignee_handle: string | null; assignee_pane_key: string | null }, @@ -19,7 +8,7 @@ function hasLifecycleAuthority( ): boolean { if (dispatch.assignee_pane_key) { return Boolean( - msg.sender_pane_key && isSamePane(dispatch.assignee_pane_key, msg.sender_pane_key) + msg.sender_pane_key && isEquivalentPaneKey(dispatch.assignee_pane_key, msg.sender_pane_key) ) } // Why: rows created before pane identity existed can only use the exact diff --git a/src/main/runtime/rpc/orchestration-legacy-process-identity.ts b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts index 35054c61279..14e4f3e46f3 100644 --- a/src/main/runtime/rpc/orchestration-legacy-process-identity.ts +++ b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts @@ -1,6 +1,8 @@ -import { parsePaneKey } from '../../../shared/stable-pane-id' +import { isEquivalentPaneKey } from '../../../shared/stable-pane-id' import { OrchestrationError } from '../orchestration/orchestration-error' +// Why: legacy rows carry nullable pane keys, and a missing key is never proof of +// identity — so absence is refused here rather than inside the shared comparison. export function equivalentLegacyPaneKey( a: string | null | undefined, b: string | null | undefined @@ -8,12 +10,7 @@ export function equivalentLegacyPaneKey( if (!a || !b) { return false } - if (a === b) { - return true - } - const aLeaf = parsePaneKey(a)?.leafId - const bLeaf = parsePaneKey(b)?.leafId - return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) + return isEquivalentPaneKey(a, b) } export function legacyReadOnlyError(): OrchestrationError { diff --git a/src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts b/src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts new file mode 100644 index 00000000000..5369e403195 --- /dev/null +++ b/src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts @@ -0,0 +1,157 @@ +/** + * Characterisation oracle for the 30-second SSH pane recovery grant in + * `OrcaRuntimeService.recoverTerminalPane`. + * + * The grant is the branch that, for a DISCONNECTED pane backed by a recently + * expired SSH lease, calls `createTerminal` to spawn a replacement shell. This + * file does not argue that the grant is wrong; it RECORDS that the branch is + * already dead at the id shapes production mints, so that deleting it is + * provably inert rather than assumed to be. Test 1 is written to survive that + * deletion unchanged: `terminal_not_recoverable` is the answer both before and + * after, and the anti-vacuity assertions pin that control genuinely reaches the + * grant gate instead of bailing out earlier. + * + * Why the branch is dead: the gate compares `pty.ptyId` (runtime, APP form — + * `SshPtyProvider` hands ids back through `toAppSshPtyId`, and that is the id + * `ipc/pty.ts` passes to `runtime.registerPty`) against `lease.ptyId` (durable, + * RELAY-native — `persistence.ts` normalizes every stored lease through + * `toRelaySshPtyId`). The comparison at `getRecentExpiredSshLease` is a raw + * `===` with no namespace normalization, so the two strings can never be equal. + * + * Honest limit on the claim. For an SSH pane the refusal is unreachable BY + * CONSTRUCTION: an app-form id always carries the `ssh:@@` prefix that + * the relay-native lease id, by definition, does not. For a LOCAL pane the + * argument is weaker — a local pty id is not namespaced, so the gate is + * unreachable only up to a random-UUID string collision with a stored lease id + * that also matches the worktree, tab and leaf. That is not a construction + * proof, just an astronomically unlikely one. Do not overclaim it. + */ +import { describe, expect, it, vi } from 'vitest' +import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' +import { OrcaRuntimeService } from './orca-runtime' + +const CONNECTION_ID = 'ssh-target-1' +const RELAY_PTY_ID = 'pty-7' +// Both id forms come from the production helpers so this oracle tracks the real +// namespace split rather than restating it as two hand-typed literals. +const APP_PTY_ID = toAppSshPtyId(CONNECTION_ID, RELAY_PTY_ID) +const STORED_LEASE_PTY_ID = toRelaySshPtyId(CONNECTION_ID, APP_PTY_ID) + +const REPO_ID = 'repo-1' +const WORKTREE_PATH = '/tmp/orca-grant-reachability' +const WORKTREE_ID = `${REPO_ID}::${WORKTREE_PATH}` +const TAB_ID = 'tab-grant' +// Persistence drops any leafId that is not a terminal leaf UUID, and the gate +// compares it, so the lease only qualifies with a real one. +const LEAF_ID = '11111111-1111-4111-8111-111111111111' +const PANE_KEY = `${TAB_ID}:${LEAF_ID}` + +function createStore() { + const now = Date.now() + return { + getRepo: (id: string) => createStore.repos.find((repo) => repo.id === id), + getRepos: () => createStore.repos, + getAllWorktreeMeta: () => ({ + [WORKTREE_ID]: { + displayName: 'grant', + comment: '', + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + linkedGitLabMR: null, + linkedGitLabIssue: null, + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + lastActivityAt: 0 + } + }), + getWorktreeMeta: () => undefined, + getSettings: () => ({ + workspaceDir: '/tmp/workspaces', + nestWorkspaces: false, + refreshLocalBaseRefOnWorktreeCreate: false, + branchPrefix: 'none', + branchPrefixCustom: '' + }), + getProjects: () => [], + getSparsePresets: () => [], + // Seeded to qualify on EVERY predicate of getRecentExpiredSshLease except + // the ptyId comparison: expired, same worktree/tab/leaf, inside the grace. + getSshRemotePtyLeases: () => [ + { + targetId: CONNECTION_ID, + ptyId: STORED_LEASE_PTY_ID, + worktreeId: WORKTREE_ID, + tabId: TAB_ID, + leafId: LEAF_ID, + state: 'expired' as const, + createdAt: now, + updatedAt: now + } + ] + } +} +createStore.repos = [ + { id: REPO_ID, path: WORKTREE_PATH, displayName: 'repo', badgeColor: 'blue', addedAt: 1 } +] + +/** Register one SSH-owned pty under `runtimePtyId`, then kill its shell. */ +function createDisconnectedPane(runtimePtyId: string) { + const runtime = new OrcaRuntimeService(createStore() as never) + runtime.registerPty(runtimePtyId, WORKTREE_ID, CONNECTION_ID, { + tabId: TAB_ID, + leafId: LEAF_ID + }) + const handle = runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).handle + runtime.onPtyExit(runtimePtyId, -1) + return { runtime, handle } +} + +describe('SSH pane recovery grant reachability', () => { + it('refuses a pane whose SSH shell died, at the id shapes production actually mints', async () => { + // The split itself, named: the durable lease id and the runtime pty id for + // the SAME shell are different strings. + expect(STORED_LEASE_PTY_ID).not.toBe(APP_PTY_ID) + + const { runtime, handle } = createDisconnectedPane(APP_PTY_ID) + const createTerminal = vi.spyOn(runtime, 'createTerminal') + + // Anti-vacuity: control must actually arrive at the grant gate. A truthy + // handle rules out an early `terminal_not_found`; connected === false rules + // out the still-connected branch that also throws not_recoverable. + expect(handle).toBeTruthy() + expect(runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).connected).toBe(false) + + await expect(runtime.recoverTerminalPane(PANE_KEY, WORKTREE_ID, handle)).rejects.toThrow( + 'terminal_not_recoverable' + ) + expect(createTerminal).not.toHaveBeenCalled() + }) + + // INVERTED BY THE DELETION, and that inversion is the point. + // + // Before the grant was removed, this case RESOLVED with a replacement: the same seeded lease, + // with the runtime pty registered under the bare relay id instead of the app-form one, opened + // the gate. That is what made the case above attributable to the namespace and to nothing else — + // the lease qualified on state, worktree, tab, leaf and grace window, and only the id comparison + // stood between it and a spawn. + // + // The shape was never reachable in production (a lease implies a truthy connectionId, which + // implies an app-form id), so flipping it to a refusal changes no real behaviour — which is + // exactly what "the deletion is inert" means. Kept rather than dropped so the counterfactual + // stays on record: no id shape authorizes a spawn now, not merely the one production mints. + it('refuses even the id shape that used to open the grant', async () => { + const { runtime, handle } = createDisconnectedPane(RELAY_PTY_ID) + const createTerminal = vi.spyOn(runtime, 'createTerminal') + + expect(handle).toBeTruthy() + expect(runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).connected).toBe(false) + + await expect(runtime.recoverTerminalPane(PANE_KEY, WORKTREE_ID, handle)).rejects.toThrow( + 'terminal_not_recoverable' + ) + expect(createTerminal).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts b/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts index 5cbb967da76..9ca0c021333 100644 --- a/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts +++ b/src/main/runtime/terminal-list-stale-leaf-liveness.test.ts @@ -148,6 +148,23 @@ describe('listTerminals liveness truth for restored leaves', () => { }) }) + it('keeps a leaf connected when the snapshot missed it but provider liveness is unknown', async () => { + const runtime = makeRuntimeWithLeaf({ + leafPtyId: 'pty-provider-disconnected', + controllerSessions: [], + hasPty: () => null + }) + + const { terminals } = await runtime.listTerminals(`id:${WORKTREE_ID}`) + + expect(terminals).toHaveLength(1) + expect(terminals[0]).toMatchObject({ + ptyId: 'pty-provider-disconnected', + connected: true, + writable: true + }) + }) + it('does not demote remote-runtime-scoped leaves the local inventory never covers', async () => { const runtime = makeRuntimeWithLeaf({ leafPtyId: 'remote:env-1@@term_abc', diff --git a/src/main/runtime/workspace-namespace-terminal-identity.test.ts b/src/main/runtime/workspace-namespace-terminal-identity.test.ts new file mode 100644 index 00000000000..d3e1a3d19bc --- /dev/null +++ b/src/main/runtime/workspace-namespace-terminal-identity.test.ts @@ -0,0 +1,365 @@ +import { describe, expect, it } from 'vitest' +import { FLOATING_TERMINAL_WORKTREE_ID, getDefaultWorkspaceSession } from '../../shared/constants' +import { makePaneKey } from '../../shared/stable-pane-id' +import type { WorkspaceSessionState } from '../../shared/types' +import { FOLDER_WORKSPACE_INSTANCE_SEPARATOR } from '../../shared/worktree-id' +import { OrcaRuntimeService } from './orca-runtime' + +/** + * Journey: git-worktree, folder-workspace and floating namespaces must all resolve + * terminal identity from one stable host-local workspace id. Neither the filesystem + * path, the client's repository id, nor the caller's target id may stand in for it. + * Drive-letter and UNC ids normalize host-independently and are covered here; the + * Windows PTY lifecycle behind them is not. + */ + +const GIT_REPO_ID = 'repo-git' +const GIT_ROOT_PATH = '/tmp/ns/checkout' +const GIT_ROOT_ID = `${GIT_REPO_ID}::${GIT_ROOT_PATH}` +const GIT_CHILD_PATH = `${GIT_ROOT_PATH}/.claude/worktrees/child` +const GIT_CHILD_ID = `${GIT_REPO_ID}::${GIT_CHILD_PATH}` + +const FOLDER_REPO_ID = 'repo-folder' +const FOLDER_PATH = '/tmp/ns/folder' +const FOLDER_ROOT_ID = `${FOLDER_REPO_ID}::${FOLDER_PATH}` +const FOLDER_A = `${FOLDER_ROOT_ID}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa` +const FOLDER_B = `${FOLDER_ROOT_ID}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb` + +// Same directory, re-added as a second project: only the repo id differs. +const ALIAS_REPO_ID = 'repo-folder-alias' +const ALIAS_ID = `${ALIAS_REPO_ID}::${FOLDER_PATH}` + +const PTY_GIT_ROOT = `${GIT_ROOT_ID}@@pty-git-root` +const PTY_GIT_CHILD = `${GIT_CHILD_ID}@@pty-git-child` +const PTY_A = `${FOLDER_A}@@pty-a` +const PTY_B = `${FOLDER_B}@@pty-b` +const PTY_FOLDER_ROOT = `${FOLDER_ROOT_ID}@@pty-folder-root` +const PTY_ALIAS = `${ALIAS_ID}@@pty-alias` +const PTY_FLOATING = 'floating-pty' + +const LEAF_A = 'a1111111-1111-4111-8111-111111111111' +const LEAF_B = 'b2222222-2222-4222-8222-222222222222' + +const REPOS = [ + { id: GIT_REPO_ID, path: GIT_ROOT_PATH, displayName: 'checkout', badgeColor: 'blue', addedAt: 1 }, + { + id: FOLDER_REPO_ID, + path: FOLDER_PATH, + displayName: 'folder', + badgeColor: 'green', + addedAt: 2, + kind: 'folder' + }, + { + id: ALIAS_REPO_ID, + path: FOLDER_PATH, + displayName: 'folder-alias', + badgeColor: 'red', + addedAt: 3, + kind: 'folder' + } +] as const + +const ALL_WORKSPACE_IDS = [GIT_ROOT_ID, GIT_CHILD_ID, FOLDER_ROOT_ID, FOLDER_A, FOLDER_B, ALIAS_ID] + +type ControllerSession = { + id: string + worktreeId?: string + cwd?: string + title?: string + incarnationId?: string + terminalHandle?: string +} + +type RuntimeInternals = { + buildResolvedWorktreeFromId: (worktreeId: string) => unknown + refreshPtyWorktreeRecordsWithControllerInventory: ( + resolvedWorktrees: unknown[], + targetWorktreeId?: string | null + ) => Promise<{ livePtyIds: Set } | null> + getLivePtyIdsForWorktree: (worktreeId: string) => Set + hasExactPersistedTerminalSurfaceIdentity: (expected: { + worktreeId: string + tabId: string + leafId: string + ptyId: string + incarnationId: string + }) => boolean + ptysById: Map + recordPtyWorktree: (ptyId: string, worktreeId: string, state?: Record) => unknown +} + +function createRuntimeInternals(options: { + session?: WorkspaceSessionState + sessions?: ControllerSession[] +}): RuntimeInternals { + const meta = Object.fromEntries(ALL_WORKSPACE_IDS.map((id) => [id, { hostId: 'local' }])) + const store = { + getRepos: () => REPOS, + getRepo: (id: string) => REPOS.find((repo) => repo.id === id), + getAllWorktreeMeta: () => meta, + getWorktreeMeta: (worktreeId: string) => meta[worktreeId], + setWorktreeMeta: (worktreeId: string, patch: Record) => { + meta[worktreeId] = { ...meta[worktreeId], ...patch } + return meta[worktreeId] + }, + getWorkspaceSession: () => options.session ?? getDefaultWorkspaceSession(), + setWorkspaceSession: () => {}, + flushOrThrow: () => {} + } as never + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + stopAndWait: async () => true, + getForegroundProcess: async () => null, + listProcesses: async () => options.sessions ?? [] + } as never) + return runtime as unknown as RuntimeInternals +} + +function resolveWorktrees(internals: RuntimeInternals, ids: readonly string[]): unknown[] { + return ids.map((id) => internals.buildResolvedWorktreeFromId(id)) +} + +/** Persisted layout binding each folder sibling's own PTY to its own tab and leaf. */ +function persistedFolderSiblingSession(): WorkspaceSessionState { + return { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [FOLDER_A]: [{ id: 'tab-a', worktreeId: FOLDER_A, title: 'A' }], + [FOLDER_B]: [{ id: 'tab-b', worktreeId: FOLDER_B, title: 'B' }] + } as never, + terminalLayoutsByTabId: { + 'tab-a': { root: null, activeLeafId: LEAF_A, ptyIdsByLeafId: { [LEAF_A]: PTY_A } }, + 'tab-b': { root: null, activeLeafId: LEAF_B, ptyIdsByLeafId: { [LEAF_B]: PTY_B } } + } as never, + terminalPtyIncarnationsByPaneKey: { + [makePaneKey('tab-a', LEAF_A)]: 'inc-a', + [makePaneKey('tab-b', LEAF_B)]: 'inc-b' + } + } +} + +describe('workspace namespaces keep terminal identity host-local and per-workspace', () => { + describe('git worktree namespace', () => { + it('binds a nested worktree PTY to the nested worktree, not the targeted parent', async () => { + // cwd alone cannot separate a parent checkout from a worktree nested inside it. + const internals = createRuntimeInternals({ + sessions: [{ id: 'unattributed-pty', cwd: `${GIT_CHILD_PATH}/src`, title: 'child work' }] + }) + const resolved = resolveWorktrees(internals, [GIT_ROOT_ID, GIT_CHILD_ID]) + + const scopedToParent = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + GIT_ROOT_ID + ) + const scopedToChild = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + GIT_CHILD_ID + ) + + expect([...(scopedToParent?.livePtyIds ?? [])]).toEqual([]) + expect([...(scopedToChild?.livePtyIds ?? [])]).toEqual(['unattributed-pty']) + expect(internals.ptysById.get('unattributed-pty')?.worktreeId).toBe(GIT_CHILD_ID) + }) + + it('keeps a checkout and the worktree nested inside it in separate namespaces', async () => { + const internals = createRuntimeInternals({ + sessions: [ + { id: PTY_GIT_ROOT, worktreeId: GIT_ROOT_ID, cwd: GIT_ROOT_PATH, title: 'git' }, + { id: PTY_GIT_CHILD, worktreeId: GIT_CHILD_ID, cwd: GIT_CHILD_PATH, title: 'child' } + ] + }) + const resolved = resolveWorktrees(internals, [GIT_ROOT_ID, GIT_CHILD_ID]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + + expect(internals.ptysById.get(PTY_GIT_ROOT)?.worktreeId).toBe(GIT_ROOT_ID) + expect(internals.ptysById.get(PTY_GIT_CHILD)?.worktreeId).toBe(GIT_CHILD_ID) + expect([...internals.getLivePtyIdsForWorktree(GIT_ROOT_ID)]).toEqual([PTY_GIT_ROOT]) + expect([...internals.getLivePtyIdsForWorktree(GIT_CHILD_ID)]).toEqual([PTY_GIT_CHILD]) + }) + }) + + describe('folder workspace namespace', () => { + it('does not let the caller target stand in for an attributed PTY owner', async () => { + const internals = createRuntimeInternals({ + sessions: [{ id: PTY_B, worktreeId: FOLDER_B, cwd: FOLDER_PATH, title: 'b' }] + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + const scopedToA = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_A + ) + + expect([...(scopedToA?.livePtyIds ?? [])]).toEqual([]) + expect(internals.ptysById.get(PTY_B)?.worktreeId).toBe(FOLDER_B) + }) + + it('keeps each sibling terminal bound to its own workspace across a reconnect', async () => { + const sessions: ControllerSession[] = [ + { + id: PTY_A, + worktreeId: FOLDER_A, + cwd: FOLDER_PATH, + title: 'a', + incarnationId: 'inc-a', + terminalHandle: 'term_a' + }, + { + id: PTY_B, + worktreeId: FOLDER_B, + cwd: FOLDER_PATH, + title: 'b', + incarnationId: 'inc-b', + terminalHandle: 'term_b' + } + ] + const internals = createRuntimeInternals({ + session: persistedFolderSiblingSession(), + sessions + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B]) + + // Reconnect resync: each workspace re-lists the same controller inventory. + const afterA = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_A + ) + const afterB = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_B + ) + + expect([...(afterA?.livePtyIds ?? [])]).toEqual([PTY_A]) + expect([...(afterB?.livePtyIds ?? [])]).toEqual([PTY_B]) + expect(internals.ptysById.get(PTY_A)?.worktreeId).toBe(FOLDER_A) + expect(internals.ptysById.get(PTY_B)?.worktreeId).toBe(FOLDER_B) + expect( + internals.hasExactPersistedTerminalSurfaceIdentity({ + worktreeId: FOLDER_B, + tabId: 'tab-b', + leafId: LEAF_B, + ptyId: PTY_B, + incarnationId: 'inc-b' + }) + ).toBe(true) + // A sibling must never satisfy the other's persisted surface identity. + expect( + internals.hasExactPersistedTerminalSurfaceIdentity({ + worktreeId: FOLDER_A, + tabId: 'tab-b', + leafId: LEAF_B, + ptyId: PTY_B, + incarnationId: 'inc-b' + }) + ).toBe(false) + }) + + it('does not treat one directory as one namespace across two projects', async () => { + // Same directory added twice: the ids differ only by the client's repo id. + const internals = createRuntimeInternals({ + sessions: [ + { id: PTY_A, worktreeId: FOLDER_A, cwd: FOLDER_PATH, title: 'a' }, + { id: PTY_FOLDER_ROOT, worktreeId: FOLDER_ROOT_ID, cwd: FOLDER_PATH, title: 'root' }, + { id: PTY_ALIAS, worktreeId: ALIAS_ID, cwd: FOLDER_PATH, title: 'alias' } + ] + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B, FOLDER_ROOT_ID, ALIAS_ID]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + + expect(internals.ptysById.get(PTY_A)?.worktreeId).toBe(FOLDER_A) + expect(internals.ptysById.get(PTY_FOLDER_ROOT)?.worktreeId).toBe(FOLDER_ROOT_ID) + expect(internals.ptysById.get(PTY_ALIAS)?.worktreeId).toBe(ALIAS_ID) + expect([...internals.getLivePtyIdsForWorktree(ALIAS_ID)]).toEqual([PTY_ALIAS]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_ROOT_ID)]).toEqual([PTY_FOLDER_ROOT]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_A)]).toEqual([PTY_A]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_B)]).toEqual([]) + }) + }) + + describe('floating workspace namespace', () => { + it('never lets a repo workspace claim a floating terminal or vice versa', async () => { + const internals = createRuntimeInternals({ + sessions: [ + { + id: PTY_FLOATING, + worktreeId: FLOATING_TERMINAL_WORKTREE_ID, + cwd: FOLDER_PATH, + title: 'floating' + }, + { id: PTY_A, worktreeId: FOLDER_A, cwd: FOLDER_PATH, title: 'a' } + ] + }) + const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B]) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved) + const scopedToA = await internals.refreshPtyWorktreeRecordsWithControllerInventory( + resolved, + FOLDER_A + ) + + // The floating id carries no repo/path pair, so it can only equal itself. + expect([...(scopedToA?.livePtyIds ?? [])]).toEqual([PTY_A]) + expect(internals.ptysById.get(PTY_FLOATING)?.worktreeId).toBe(FLOATING_TERMINAL_WORKTREE_ID) + expect([...internals.getLivePtyIdsForWorktree(FLOATING_TERMINAL_WORKTREE_ID)]).toEqual([ + PTY_FLOATING + ]) + expect([...internals.getLivePtyIdsForWorktree(FOLDER_A)]).toEqual([PTY_A]) + }) + + it('does not resolve a floating terminal onto a workspace that shares its cwd', () => { + const internals = createRuntimeInternals({}) + internals.recordPtyWorktree(PTY_FLOATING, FLOATING_TERMINAL_WORKTREE_ID, { connected: true }) + internals.recordPtyWorktree(PTY_A, FOLDER_A, { connected: true }) + + expect([...internals.getLivePtyIdsForWorktree(FOLDER_ROOT_ID)]).toEqual([]) + expect([...internals.getLivePtyIdsForWorktree(FLOATING_TERMINAL_WORKTREE_ID)]).toEqual([ + PTY_FLOATING + ]) + }) + }) + + // Only the id-normalization half of the Windows namespaces is host-independent. + // Spawning and reconnecting a real drive/UNC PTY still needs a Windows host. + describe('drive-letter and UNC namespaces (id normalization only)', () => { + it('folds drive spelling and WSL UNC aliases but keeps distinct roots apart', () => { + const internals = createRuntimeInternals({}) + internals.recordPtyWorktree('pty-c', 'repo-win::C:\\ws\\proj', { connected: true }) + internals.recordPtyWorktree('pty-d', 'repo-win::D:\\ws\\proj', { connected: true }) + internals.recordPtyWorktree('pty-share', 'repo-win::\\\\server\\share\\proj', { + connected: true + }) + internals.recordPtyWorktree('pty-other-share', 'repo-win::\\\\server\\other\\proj', { + connected: true + }) + internals.recordPtyWorktree('pty-wsl', 'repo-win::\\\\wsl$\\Ubuntu\\home\\me', { + connected: true + }) + + expect([...internals.getLivePtyIdsForWorktree('repo-win::c:/ws/proj')]).toEqual(['pty-c']) + expect([...internals.getLivePtyIdsForWorktree('repo-win::D:/ws/proj')]).toEqual(['pty-d']) + expect([...internals.getLivePtyIdsForWorktree('repo-win::\\\\server\\share\\proj')]).toEqual([ + 'pty-share' + ]) + expect([ + ...internals.getLivePtyIdsForWorktree('repo-win::\\\\wsl.localhost\\ubuntu\\home\\me') + ]).toEqual(['pty-wsl']) + }) + + it('keeps a folder workspace instance on a drive path distinct from its root', () => { + const driveRoot = 'repo-win::C:\\ws\\proj' + const driveInstance = `${driveRoot}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}dddddddd-dddd-4ddd-8ddd-dddddddddddd` + const internals = createRuntimeInternals({}) + internals.recordPtyWorktree('pty-drive-root', driveRoot, { connected: true }) + internals.recordPtyWorktree('pty-drive-instance', driveInstance, { connected: true }) + + expect([...internals.getLivePtyIdsForWorktree(driveRoot)]).toEqual(['pty-drive-root']) + expect([...internals.getLivePtyIdsForWorktree(driveInstance)]).toEqual(['pty-drive-instance']) + }) + }) +}) diff --git a/src/main/runtime/workspace-session-terminal-membership-authority.ts b/src/main/runtime/workspace-session-terminal-membership-authority.ts index 9dace4c8cb1..9509f14eb26 100644 --- a/src/main/runtime/workspace-session-terminal-membership-authority.ts +++ b/src/main/runtime/workspace-session-terminal-membership-authority.ts @@ -161,6 +161,30 @@ export function advanceTerminalTopologyRevision( } } +/** + * The tab whose live layout holds this leaf. Only the leaf half of a pane key is stable — breaking + * a pane out into its own tab moves the leaf and leaves any stored tabId naming the tab it left. + */ +export function findTerminalTabIdForLeaf( + session: WorkspaceSessionState | undefined, + leafId: string +): string | undefined { + // Only a still-existing tab counts: a layout entry outlives its tab, and binding a live shell to + // a deleted one registers a pane under a ghost. Scanned across worktrees rather than under one + // key so the answer cannot hinge on a worktreeId matching exactly (`::workspace:` suffixes). + const liveTabIds = new Set( + Object.values(session?.tabsByWorktree ?? {}).flatMap((tabs) => tabs.map((tab) => tab.id)) + ) + for (const [tabId, layout] of Object.entries(session?.terminalLayoutsByTabId ?? {})) { + const leafIds = new Set() + collectLeafIds(layout.root, leafIds) + if (liveTabIds.has(tabId) && leafIds.has(leafId)) { + return tabId + } + } + return undefined +} + export function hasHostAuthoritativeTerminalMembership( session: WorkspaceSessionState | undefined, worktreeId: string diff --git a/src/main/ssh-pane-binding-partition.test.ts b/src/main/ssh-pane-binding-partition.test.ts new file mode 100644 index 00000000000..a630f62a941 --- /dev/null +++ b/src/main/ssh-pane-binding-partition.test.ts @@ -0,0 +1,399 @@ +/** + * STA-3077 step P: an SSH pane's durable binding must live in ONE partition. + * + * Today it lives in two. Main's spawn writes `ssh:` + * (ipc/pty.ts persistPtyBinding(binding, toSshExecutionHostId(connectionId))), + * the relay's reattach write passes no hostId and lands in `local` + * (ssh-relay-session.ts restoreReattachedPtyRuntime), and the renderer keeps SSH + * worktrees in `local` on purpose (buildHostIdByWorktreeId). Supersession then + * reads a binding no live writer maintains, so it compares the arriving lease + * against a stale pty id, bails, and leaves the predecessor live. That is the + * reported 2 -> 19 -> 20 mechanism. + * + * These oracles assert cardinality and identity — one pane, one live claim, and + * the surviving claim is the shell the pane is bound to — never which function + * ran, so they stay valid under any single-accessor implementation. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { getDefaultPersistedState, getDefaultWorkspaceSession } from '../shared/constants' +import { LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId } from '../shared/execution-host' +import { toAppSshPtyId, toRelaySshPtyId } from '../shared/ssh-pty-id' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('./telemetry/client', () => ({ track: vi.fn() })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TARGET = 'ssh-target-1' +const SSH_PARTITION = toSshExecutionHostId(TARGET) +const WORKTREE = 'repo-1:wt-1' +const TAB = 'tab-1' +/** Must be a real layout leaf UUID — the store drops any other spelling. */ +const LEAF = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + +const appPtyId = (relayPtyId: string): string => toAppSshPtyId(TARGET, relayPtyId) + +async function createStore(state: Record = {}) { + mkdirSync(testState.dir, { recursive: true }) + writeFileSync( + join(testState.dir, 'orca-data.json'), + JSON.stringify({ ...getDefaultPersistedState(testState.dir), ...state }), + 'utf-8' + ) + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() +} + +type TestStore = Awaited> + +/** One SSH pane, bound to `relayPtyId`, as a partition stores it. */ +function paneSession(relayPtyId: string) { + const ptyId = appPtyId(relayPtyId) + return { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { + [WORKTREE]: [ + { + id: TAB, + ptyId, + worktreeId: WORKTREE, + title: 'Terminal 1', + defaultTitle: 'Terminal 1', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + [TAB]: { + root: { type: 'leaf' as const, leafId: LEAF }, + activeLeafId: LEAF, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF]: ptyId } + } + } + } +} + +/** session:set for an SSH worktree — buildHostIdByWorktreeId sends it to `local`. */ +function rendererPublishesPane(store: TestStore, relayPtyId: string): void { + store.setWorkspaceSession(paneSession(relayPtyId) as never, LOCAL_EXECUTION_HOST_ID) +} + +/** A mid-session write into `ssh:` — orphan adoption still targets that partition, so the + * one-time load fold cannot be the only thing keeping the two homes from reappearing. */ +function somethingRewritesTheSshPartition(store: TestStore, relayPtyId: string): void { + store.setWorkspaceSession(paneSession(relayPtyId) as never, SSH_PARTITION) +} + +/** ssh-relay-session.ts:2504 — the reattach bind. No hostId, refuses to create. */ +function relayReattachBindsPane(store: TestStore, relayPtyId: string): boolean | null { + return store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: appPtyId(relayPtyId), + incarnationId: `inc-${relayPtyId}`, + mayCreate: false + }) +} + +/** ipc/pty.ts:6493 — the spawn's lease upsert, ahead of its binding write. */ +function sshSpawnUpsertsLease(store: TestStore, relayPtyId: string): void { + store.upsertSshRemotePtyLease({ + targetId: TARGET, + ptyId: toRelaySshPtyId(TARGET, appPtyId(relayPtyId)), + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + state: 'attached', + lastAttachedAt: Date.now() + }) +} + +function liveLeaseIdsForPane(store: TestStore): string[] { + return store + .getSshRemotePtyLeases(TARGET) + .filter( + (lease) => + lease.tabId === TAB && + lease.leafId === LEAF && + lease.state !== 'terminated' && + lease.state !== 'expired' + ) + .map((lease) => lease.ptyId) + .sort() +} + +/** The argument text of every `callee(...)` call in a production source file. */ +function callArgumentsIn(source: string, callee: string): string[] { + const calls: string[] = [] + for (let index = source.indexOf(`${callee}(`); index !== -1; ) { + let cursor = index + callee.length + const start = cursor + 1 + for (let depth = 0; cursor < source.length; cursor += 1) { + if (source[cursor] === '(') { + depth += 1 + } else if (source[cursor] === ')' && --depth === 0) { + break + } + } + calls.push(source.slice(start, cursor)) + index = source.indexOf(`${callee}(`, cursor) + } + return calls +} + +/** On-disk state from an earlier session: both partitions name the same shell, + * because main's spawn wrote `ssh:` and the renderer published `local`. + * They diverge as soon as the relay's reattach write updates only `local`. */ +function diskAfterEarlierSession(relayPtyId: string) { + return { + workspaceSession: paneSession(relayPtyId), + workspaceSessionsByHostId: { [SSH_PARTITION]: paneSession(relayPtyId) } + } +} + +beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-sta3077-partition-')) +}) + +describe('STA-3077 step P: one pane, one live claim across partitions', () => { + // The pane is bound to pty-2 in `local` (the relay's reattach write put it + // there); the SSH partition still names the predecessor pty-1. Supersession + // reads ssh-first, sees pty-1 != pty-2, and bails — both shells stay claimed. + it('supersedes the predecessor when the pane is bound in the local partition', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + sshSpawnUpsertsLease(store, 'pty-1') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + + sshSpawnUpsertsLease(store, 'pty-2') + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) + + // reattachKnownPtys calls this first, so a wrong winner here is what fans the + // reconnect out over a dead id and grafts a pane the user never opened. + it('keeps the lease the live pane binding names when healing duplicates', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + sshSpawnUpsertsLease(store, 'pty-1') + sshSpawnUpsertsLease(store, 'pty-2') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + + await store.supersedeDuplicatePaneLeases(TARGET) + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) + + // Isolates the reader from the one-time load fold. Without this clause the fold masks the + // partition preference — it deletes the divergent copy at boot, so restoring the ssh-first + // hedge stays green and the reader guard ships unproven. Anything that writes `ssh:` + // after load (orphan adoption still does) would then revive the defect inside one session. + it('supersedes the predecessor when the ssh partition is rewritten mid-session', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + sshSpawnUpsertsLease(store, 'pty-1') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + somethingRewritesTheSshPartition(store, 'pty-1') + + sshSpawnUpsertsLease(store, 'pty-2') + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) + + // The reported growth: live claims must not scale with reconnect count. + it('holds the live claim count flat across ten reconnects of one pane', async () => { + const store = await createStore(diskAfterEarlierSession('pty-0')) + sshSpawnUpsertsLease(store, 'pty-0') + + for (let reconnect = 1; reconnect <= 10; reconnect += 1) { + expect(relayReattachBindsPane(store, `pty-${reconnect}`)).toBe(true) + sshSpawnUpsertsLease(store, `pty-${reconnect}`) + } + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-10']) + }) +}) + +describe('STA-3077: a decision made from one plane only mutates that plane', () => { + // Arbitration ranks using the desktop plane's binding. The headless plane's binding never gets a + // vote, so deleting it here would strand a shell its owner may still be using — and that owner + // would then have no durable record to reattach by. An explicit expiry is different: the pty is + // gone for everyone, and that path still scrubs both. + it('leaves the headless plane binding intact when supersession retires its lease', async () => { + const sshSession = paneSession('pty-old') as unknown as Record + const store = await createStore({ + workspaceSession: paneSession('pty-new'), + workspaceSessionsByHostId: { [SSH_PARTITION]: sshSession } + }) + sshSpawnUpsertsLease(store, 'pty-old') + + // Local names pty-new, so arbitration expires pty-old without the other plane being consulted. + sshSpawnUpsertsLease(store, 'pty-new') + + expect(liveLeaseIdsForPane(store)).toEqual(['pty-new']) + expect( + store.getWorkspaceSession(SSH_PARTITION).terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF] + ).toBe(appPtyId('pty-old')) + }) +}) + +describe('STA-3077: a pane moved between tabs still supersedes its own predecessor', () => { + // The reported cardinality growth, in its surviving form. A lease freezes its tabId at write + // time. Break the pane out into a new tab and its next lease carries the NEW tab, so matching + // siblings on tabId means the two leases for one pane never meet — the predecessor is never + // superseded and the count grows on every reconnect, exactly as reported. + function leaseInTab(store: TestStore, relayPtyId: string, tabId: string): void { + store.upsertSshRemotePtyLease({ + targetId: TARGET, + ptyId: toRelaySshPtyId(TARGET, appPtyId(relayPtyId)), + worktreeId: WORKTREE, + tabId, + leafId: LEAF, + state: 'attached', + lastAttachedAt: Date.now() + }) + } + const liveForLeaf = (store: TestStore): string[] => + store + .getSshRemotePtyLeases(TARGET) + .filter((l) => l.leafId === LEAF && l.state !== 'terminated' && l.state !== 'expired') + .map((l) => l.ptyId) + .sort() + + it('supersedes a predecessor whose lease names the tab the pane left', async () => { + const store = await createStore() + leaseInTab(store, 'pty-old', TAB) + + leaseInTab(store, 'pty-new', 'tab-moved-to') + + expect(liveForLeaf(store)).toEqual(['pty-new']) + }) + + it('holds the count flat across ten reconnects that each land in a new tab', async () => { + const store = await createStore() + leaseInTab(store, 'pty-0', TAB) + for (let n = 1; n <= 10; n += 1) { + leaseInTab(store, `pty-${n}`, `tab-${n}`) + } + + expect(liveForLeaf(store)).toEqual(['pty-10']) + }) +}) + +describe('STA-3077: arbitration follows the pane, not the tab it was written in', () => { + // A lease freezes its tabId when written; `detachTerminalPaneToTab` then moves the live pane and + // its PTY into a new tab. Looking the binding up under the frozen tab finds nothing, so the bound + // shell loses to recency and supersession retires the pane's OWN shell in favour of a stale one. + it('finds the binding after the pane has been moved to another tab', async () => { + const MOVED_TAB = 'tab-moved-to' + const store = await createStore() + // The renderer publishes the pane in its NEW tab; the lease still names the old one. + store.setWorkspaceSession( + { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [WORKTREE]: [{ id: MOVED_TAB, worktreeId: WORKTREE }] }, + terminalLayoutsByTabId: { + [MOVED_TAB]: { + root: { type: 'leaf' as const, leafId: LEAF }, + activeLeafId: LEAF, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF]: appPtyId('pty-bound') } + } + } + } as never, + LOCAL_EXECUTION_HOST_ID + ) + sshSpawnUpsertsLease(store, 'pty-bound') + // A newer, unbound lease arrives for the same pane under the lease's frozen tab. + sshSpawnUpsertsLease(store, 'pty-newer') + + await store.supersedeDuplicatePaneLeases(TARGET) + + // The bound shell must win. Keyed on the frozen tab, it is invisible and recency retires it. + expect(liveLeaseIdsForPane(store)).toEqual(['pty-bound']) + }) +}) + +describe('STA-3077 step P: the desktop plane resolves from its own home', () => { + // INVERTED. This clause used to require the two partitions to AGREE after load, which assumed + // `ssh:` was a stale spill of this plane's state. It is not — it is the headless/CLI + // plane's own home, which that plane writes and reads deliberately (STA-3463, STA-3465). The + // property that actually matters is narrower and stronger: what the desktop plane resolves must + // follow `local` alone, whatever the other plane holds. Asserting agreement made a migration look + // necessary that in fact erased another plane's live state. + it('resolves to the local binding regardless of what the ssh partition holds', async () => { + const store = await createStore(diskAfterEarlierSession('pty-1')) + + rendererPublishesPane(store, 'pty-1') + expect(relayReattachBindsPane(store, 'pty-2')).toBe(true) + // The other plane names the predecessor. It does not get a vote here — local speaking about + // this leaf IS the desktop plane's live view, and letting the other copy outrank it is the + // STA-3077 defect. Its binding to the loser is then scrubbed with every other reference to + // that retired lease, so the two planes do not drift further apart. + somethingRewritesTheSshPartition(store, 'pty-1') + sshSpawnUpsertsLease(store, 'pty-2') + + expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe( + appPtyId('pty-2') + ) + expect(liveLeaseIdsForPane(store)).toEqual(['pty-2']) + }) +}) + +describe('STA-3077 step P: every production caller names that one home', () => { + const summarize = (source: string): string => source.replace(/\s+/g, ' ').trim().slice(0, 90) + + // A guard that behaves correctly is not evidence that both writers reach it, + // and the defect IS that they disagree — so pin the call sites too. + it('has no SSH pane binding write that selects the ssh partition', () => { + const file = 'src/main/ipc/pty.ts' + const calls = callArgumentsIn(readFileSync(file, 'utf-8'), 'persistPtyBinding') + expect(calls.length, `${file} no longer writes pane bindings`).toBeGreaterThan(0) + + const partitioned = calls + .filter((call) => call.includes('toSshExecutionHostId')) + .map((call) => `${file}: ${summarize(call)}`) + + expect(partitioned).toEqual([]) + }) + + // STRENGTHENED, not relaxed. This clause used to require the relay to hold a `persistPtyBinding` + // call of its own and merely forbid an ssh-partition argument on it. Step F removed that call: + // the relay binds through the one `bindPaneShell` producer, which is what makes the superseded- + // pane fence live on reattach. Requiring ZERO direct writes here is the stronger property — a + // second bind producer is exactly the defect that let spawn and reattach disagree. + it('has no pane binding write in the relay that bypasses the one bind producer', () => { + const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + + expect(callArgumentsIn(source, 'persistPtyBinding').map(summarize)).toEqual([]) + expect(source).toContain('bindPaneShell(') + }) + + // The readers must land in the same place; one that still consults + // `ssh:` reinstates the disagreement from the other side. + it('has no stable-pane owner reader that selects the ssh partition', () => { + const source = readFileSync('src/main/ipc/pty.ts', 'utf-8') + const start = source.indexOf('function resolvePersistedStablePaneOwner') + const readers = source.slice(start, source.indexOf('type StablePaneSpawnContext')) + expect(start, 'stable-pane owner readers moved').toBeGreaterThan(0) + expect(readers).toContain('getWorkspaceSession(') + + const partitioned = readers + .split('\n') + .filter((line) => line.includes('toSshExecutionHostId')) + .map(summarize) + + expect(partitioned).toEqual([]) + }) +}) diff --git a/src/main/ssh-reattach-pane-cardinality.test.ts b/src/main/ssh-reattach-pane-cardinality.test.ts new file mode 100644 index 00000000000..8f5665981c4 --- /dev/null +++ b/src/main/ssh-reattach-pane-cardinality.test.ts @@ -0,0 +1,742 @@ +/** + * STA-3077 behavioral oracles: reconnecting an SSH workspace must not add panes + * the user never opened, and must not accumulate remote shells. + * + * These assert observable behavior, not a mechanism, so they stay valid under + * any implementation that fixes the defect. Each case names the root cause it + * pins from the #12264 diagnosis. + * + * Reported symptom: relay PTY count went 2 -> 19 -> 20 across three reconnects. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { getDefaultPersistedState } from '../shared/constants' + +const testState = { dir: '' } + +vi.mock('electron', () => ({ + app: { getPath: () => testState.dir }, + safeStorage: { isEncryptionAvailable: () => false } +})) +vi.mock('./telemetry/client', () => ({ track: vi.fn() })) +vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() })) + +const TARGET = 'ssh-target-1' +const WORKTREE = 'repo-1:wt-1' +const TAB = 'tab-1' +/** Must be a real layout leaf UUID — the store drops any other spelling. */ +const LEAF = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90' + +async function createStore(state: Record = {}) { + mkdirSync(testState.dir, { recursive: true }) + writeFileSync( + join(testState.dir, 'orca-data.json'), + JSON.stringify({ ...getDefaultPersistedState(testState.dir), ...state }), + 'utf-8' + ) + vi.resetModules() + const { Store, initDataPath } = await import('./persistence') + initDataPath() + return new Store() +} + +beforeEach(() => { + testState.dir = mkdtempSync(join(tmpdir(), 'orca-sta3077-')) +}) + +/** One pane's lease, as the reattach path records it. */ +function leaseFor(ptyId: string, updatedAt: number) { + return { + targetId: TARGET, + ptyId, + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + state: 'attached' as const, + updatedAt + } +} + +function liveLeasesForPane(store: { + getSshRemotePtyLeases: (targetId?: string) => readonly { + ptyId: string + tabId?: string + leafId?: string + state: string + }[] +}) { + return store + .getSshRemotePtyLeases(TARGET) + .filter( + (lease) => + lease.tabId === TAB && + lease.leafId === LEAF && + lease.state !== 'terminated' && + lease.state !== 'expired' + ) +} + +describe('STA-3077: one pane owns at most one live remote PTY lease', () => { + // RC1: lease uniqueness keys on (targetId, ptyId) only, so a pane whose PTY id + // changes leaves its predecessor behind with nothing to retire it. + it('does not accumulate a second live lease when one pane re-leases a new PTY id', async () => { + const store = await createStore() + + store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1)) + store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2)) + + expect(liveLeasesForPane(store)).toHaveLength(1) + }) + + // RC1: the reported 2 -> 19 -> 20 growth. Lease count must not scale with + // reconnect count for a fixed set of panes. + it('keeps live lease count flat across repeated reconnects of one pane', async () => { + const store = await createStore() + + for (let reconnect = 0; reconnect < 10; reconnect += 1) { + store.upsertSshRemotePtyLease(leaseFor(`relay-pty-${reconnect}`, reconnect + 1)) + } + + expect(liveLeasesForPane(store)).toHaveLength(1) + }) +}) + +describe('STA-3077: superseding respects the durable pane binding', () => { + // Spawn-shaped setup: the creating branches are intentionally allowed here. + function bindPaneTo(store: Awaited>, ptyId: string) { + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId, + incarnationId: `inc-${ptyId}` + }) + } + + it('scrubs the predecessor binding when the arriving lease is the bound one', async () => { + const store = await createStore() + bindPaneTo(store, 'relay-pty-a') + store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1)) + bindPaneTo(store, 'relay-pty-b') + store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2)) + + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-b']) + const layout = store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB] + expect(layout?.ptyIdsByLeafId?.[LEAF]).toBe('relay-pty-b') + }) + + // Expiring the bound predecessor here would detach a live pane, so both stay + // live and reattach arbitrates with the binding in hand. + it('defers instead of expiring the lease the pane is bound to', async () => { + const store = await createStore() + bindPaneTo(store, 'relay-pty-a') + store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1)) + store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2)) + + expect( + liveLeasesForPane(store) + .map((lease) => lease.ptyId) + .sort() + ).toEqual(['relay-pty-a', 'relay-pty-b']) + expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe( + 'relay-pty-a' + ) + + // Arbitration keeps the bound lease, not the newer one. + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1) + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-a']) + }) +}) + +describe('STA-3077: existing duplicate leases are healed, not revived', () => { + // Installs that predate pane-keyed supersession already carry the duplicates + // this bug accumulated. Preventing new ones does not help them. + it('retires every stale duplicate for a pane and keeps the newest', async () => { + const store = await createStore({ + sshRemotePtyLeases: Array.from({ length: 20 }, (_, index) => ({ + ...leaseFor(`relay-pty-${index}`, index + 1), + createdAt: index + 1 + })) + }) + + const retired = await store.supersedeDuplicatePaneLeases(TARGET) + + expect(retired).toBe(19) + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-19']) + }) + + // Recency alone would retire the lease the pane is actually bound to whenever a + // newer unbound lease exists, detaching a live pane instead of healing it. + it('keeps the durably bound lease even when an unbound one is newer', async () => { + const store = await createStore() + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-bound', + incarnationId: 'inc-bound' + }) + store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-bound', 1), createdAt: 1 }) + // Arrives later but no pane is bound to it. + store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-newer', 99), createdAt: 99 }) + + await store.supersedeDuplicatePaneLeases(TARGET) + + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-bound']) + }) + + // A retirement that is not durable must not be believed: it would read as + // retired in memory and attached on disk for the rest of the session. + it('rolls the retirement back when the durable write fails', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + // The durable write is the async twin: retirement runs on reconnect, and the sync flush + // fsyncs a multi-MB file from the main thread. Reaching for a private name is not ideal, + // but the property under test is the rollback, and it must follow whichever writer the + // retirement actually awaits. + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockRejectedValue(new Error('disk full')) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(0) + expect(liveLeasesForPane(store)).toHaveLength(2) + }) + + it('does not chase store mutations after the retirement generation is durable', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + const internal = store as unknown as { enqueueWrite: () => Promise } + const enqueueWrite = internal.enqueueWrite.bind(store) + let injectedMutation = false + const enqueueSpy = vi.spyOn(internal, 'enqueueWrite').mockImplementation(async () => { + await enqueueWrite() + if (!injectedMutation) { + injectedMutation = true + store.updateUI({ sidebarWidth: 777 }) + } + }) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1) + + expect(enqueueSpy).toHaveBeenCalledOnce() + enqueueSpy.mockRestore() + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) + + it('retries when a mutation invalidates the retirement write before rename', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + const internal = store as unknown as { writeToDiskAsync: () => Promise } + const writeToDiskAsync = internal.writeToDiskAsync.bind(store) + let injectedMutation = false + const writeSpy = vi.spyOn(internal, 'writeToDiskAsync').mockImplementation(async () => { + const write = writeToDiskAsync() + if (!injectedMutation) { + injectedMutation = true + store.updateUI({ sidebarWidth: 778 }) + } + await write + }) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1) + + expect(writeSpy).toHaveBeenCalledTimes(2) + }) + + it('does not undo a concurrent confirmation of the provisional expired state', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'expired') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('expired') + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) + + it('rolls back after a rejected concurrent attach request', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'attached') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('attached') + }) + + it('does not revive a provisional loser after a concurrent same-pane upsert', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-c', 3), createdAt: 3 }) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-c']) + }) + + it('does not restore attached state over a concurrent detach', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLeasesForShutdown(TARGET, 'detached') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('expired') + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-b')?.state + ).toBe('detached') + }) + + it('does not roll back concurrent session or lease changes after a failed write', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'terminated') + const concurrentTab = 'tab-concurrent' + const concurrentLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: concurrentTab, + leafId: concurrentLeaf, + ptyId: 'relay-pty-concurrent', + incarnationId: 'inc-concurrent' + }) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state + ).toBe('terminated') + expect( + store.getWorkspaceSession().terminalLayoutsByTabId?.[concurrentTab]?.ptyIdsByLeafId?.[ + concurrentLeaf + ] + ).toBe('relay-pty-concurrent') + }) + + it('does not restore a binding for a lease removed during a failed write', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a' + }) + delete store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF] + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + store.removeSshRemotePtyLease(TARGET, 'relay-pty-a') + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect(store.getSshRemotePtyLeases(TARGET).some((lease) => lease.ptyId === 'relay-pty-a')).toBe( + false + ) + expect( + store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)?.ptyId + ).toBeNull() + }) + + it('restores a tab binding into the moved leaf after a concurrent session replacement', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a' + }) + delete store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF] + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + expect( + store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)?.ptyId + ).toBeNull() + const movedTabId = 'tab-moved' + const replacement = structuredClone(store.getWorkspaceSession()) + const movedTab = replacement.tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB) + if (!movedTab || !replacement.terminalLayoutsByTabId?.[TAB]) { + throw new Error('expected source tab and layout') + } + movedTab.id = movedTabId + replacement.terminalLayoutsByTabId[movedTabId] = replacement.terminalLayoutsByTabId[TAB] + delete replacement.terminalLayoutsByTabId[TAB] + store.setWorkspaceSession(replacement) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect( + store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === movedTabId) + ?.ptyId + ).toBe('relay-pty-a') + await store.flushPendingOrThrowAsync({ drainToStableGeneration: false }) + }) + + it('does not restore a retired binding after the leaf is deleted', async () => { + const otherTab = 'tab-other' + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2, tabId: otherTab } + ] + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a' + }) + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: otherTab, + leafId: LEAF, + ptyId: 'relay-pty-b', + incarnationId: 'inc-b' + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.supersedeDuplicatePaneLeases(TARGET) + expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe( + undefined + ) + const replacement = structuredClone(store.getWorkspaceSession()) + replacement.tabsByWorktree![WORKTREE] = [] + replacement.terminalLayoutsByTabId = {} + store.setWorkspaceSession(replacement) + rejectWrite(new Error('disk full')) + + expect(await retirement).toBe(0) + expect(store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]).toEqual([]) + expect(store.getWorkspaceSession().terminalLayoutsByTabId).toEqual({}) + }) + + it('leaves distinct panes alone', async () => { + const otherLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2, leafId: otherLeaf } + ] + }) + + expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(0) + expect(store.getSshRemotePtyLeases(TARGET).filter((l) => l.state === 'attached')).toHaveLength( + 2 + ) + }) + + // A lease is rebuilt field by field on load, so a field added to the type but not to the + // normalizer is dropped on every boot — it would look right in memory and be gone on the next + // start, which is how a fence ships silently permitting everything. A synthesized identity is + // refused outright rather than carried: it is not stable across reconnects, so one persisted by + // another build would later read as a different shell and strand a live pane. + it('keeps a host-attested incarnation across a reload and refuses a synthesized one', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1, incarnationId: 'inc-host-a' }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2, incarnationId: 'legacy:23:0:relay-pty-b' } + ] + }) + + const byPtyId = new Map(store.getSshRemotePtyLeases(TARGET).map((l) => [l.ptyId, l])) + expect(byPtyId.get('relay-pty-a')?.incarnationId).toBe('inc-host-a') + expect(byPtyId.get('relay-pty-b')?.incarnationId).toBeUndefined() + }) +}) + +describe('SSH reconnect lease retirement batching', () => { + it('keeps concurrent terminal state when the asynchronous write fails', async () => { + const store = await createStore({ + sshRemotePtyLeases: [ + { ...leaseFor('relay-pty-a', 1), createdAt: 1 }, + { ...leaseFor('relay-pty-b', 2), createdAt: 2 } + ] + }) + let rejectWrite: (error: Error) => void = () => {} + vi.spyOn( + store as unknown as { flushDurableStateOrThrowAsync: () => Promise }, + 'flushDurableStateOrThrowAsync' + ).mockImplementation( + () => + new Promise((_, reject) => { + rejectWrite = reject + }) + ) + + const retirement = store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['relay-pty-a']) + const concurrentTab = 'tab-concurrent' + const concurrentLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d' + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: concurrentTab, + leafId: concurrentLeaf, + ptyId: 'relay-pty-concurrent', + incarnationId: 'inc-concurrent' + }) + rejectWrite(new Error('disk full')) + + await expect(retirement).rejects.toThrow('disk full') + expect( + store.getWorkspaceSession().terminalLayoutsByTabId?.[concurrentTab]?.ptyIdsByLeafId?.[ + concurrentLeaf + ] + ).toBe('relay-pty-concurrent') + }) +}) + +describe('STA-3077: reattach binds panes, it never creates them', () => { + // RC3: persistPtyBinding has four creating branches (mint tab, mint root leaf, + // split root and graft leaf, mint layout). They are load-bearing for spawn and + // wrong for reattach, where the pane either exists or is gone for good. + it('does not mint a tab for a pane that no longer exists', async () => { + const store = await createStore() + + store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a', + mayCreate: false + }) + + const session = store.getWorkspaceSession() + expect(session.tabsByWorktree?.[WORKTREE] ?? []).toHaveLength(0) + expect(session.terminalLayoutsByTabId?.[TAB]).toBeUndefined() + }) + + // Unknown is not dead: failing to resolve a pane must not be reported as + // success, and must not terminate anything. + it('reports an unresolved reattach rather than silently succeeding', async () => { + const store = await createStore() + + const bound = store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-a', + incarnationId: 'inc-a', + mayCreate: false + }) + + expect(bound).toBe(false) + }) +}) + +describe('STA-3077: exact-binding compare-and-swap', () => { + // Already correct at HEAD. Pinned so a fix cannot regress it: a stale renderer + // replay must not overwrite a binding the host has since re-admitted. + it('refuses a write whose expected binding no longer matches', async () => { + const store = await createStore() + + const stale = store.persistPtyBinding({ + worktreeId: WORKTREE, + tabId: TAB, + leafId: LEAF, + ptyId: 'relay-pty-b', + incarnationId: 'inc-b', + expectedBinding: { ptyId: 'relay-pty-a', incarnationId: 'inc-a' } + }) + + expect(stale).toBe(false) + }) +}) + +describe('STA-3077 step F: live-layout tab resolution is reattach-only', () => { + // The lease's tabId is the frozen side on reattach, so the live layout outranks it there. On a + // SPAWN the opposite holds: the caller's tabId is fresh truth and the persisted layout is the + // stale side, because the renderer publishes membership on a debounce. Breaking a pane out into + // a new tab and spawning into it inside that window would otherwise resolve back to the tab the + // pane just left, writing the durable binding and the fence under one tab while the lease and + // the runtime registration use the other — the split-coordinate defect step F exists to remove. + it('resolves the live tab only on the reattach bind, never on a spawn', async () => { + const { readFileSync } = await import('node:fs') + const relay = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + const ipc = readFileSync('src/main/ipc/pty.ts', 'utf-8') + + // The reattach holds a tabId frozen in a lease, so it must resolve the live one first. + expect(relay).toContain('resolvePaneShellTabId(') + // Resolving BEFORE the durable write is what keeps a thrown write from losing the answer and + // registering the pane in the graph under the tab it left. + expect(relay.indexOf('resolvePaneShellTabId(')).toBeLessThan(relay.indexOf('bindPaneShell({')) + // Spawn callers hold the fresh tabId; none of them may consult the debounced layout. + expect(ipc).toContain('bindPaneShell({') + expect(ipc.split('bindPaneShell({').slice(1).join('')).not.toContain('resolvePaneShellTabId') + }) +}) + +describe('STA-3077: the reattach path actually refuses to create', () => { + // This is the oracle the store-level tests could not provide: `mayCreate` + // existed and was correct, but no production caller passed it, so reattach + // still grafted panes. Pin the wiring, not just the capability. + // The reattach bind now goes through the one `bindPaneShell` producer, and it + // still refuses to create. + it('passes mayCreate:false from the SSH reattach binding write', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + const bindCall = source.slice( + source.indexOf('restoreReattachedPtyRuntime'), + source.indexOf('private async attachPtyWithRetry') + ) + expect(bindCall).toContain('bindPaneShell') + expect(bindCall).toContain('mayCreate: false') + }) + + // Strengthened, not relaxed: counting guards against calls went vacuous once + // the direct store calls disappeared, so pin both halves instead. + it('has no production persistPtyBinding caller that can create during reattach', async () => { + const { readFileSync } = await import('node:fs') + const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8') + // Every bind in the relay session is a reattach; none may grow topology. + expect(source.split('persistPtyBinding(').length - 1).toBe(0) + const binds = source.split('bindPaneShell(').length - 1 + expect(binds).toBeGreaterThan(0) + // Each `bindPaneShell(` call site must carry its own `mayCreate: false`. + const guardedBinds = source + .split('bindPaneShell(') + .slice(1) + .filter((tail) => tail.slice(0, tail.indexOf('})')).includes('mayCreate: false')).length + expect(guardedBinds).toBe(binds) + }) +}) diff --git a/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts b/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts index 9d08ac7da0f..9de9e9f1100 100644 --- a/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts +++ b/src/main/ssh/ssh-relay-orphan-abandon-paths.test.ts @@ -69,6 +69,7 @@ vi.mock('../providers/ssh-git-provider', () => ({ SshGitProvider: class MockSshGitProvider {} })) vi.mock('../ipc/pty', () => ({ + resolvePaneShellTabId: vi.fn(() => undefined), registerSshPtyProvider: vi.fn(), unregisterSshPtyProvider: vi.fn(), getSshPtyProvider: vi.fn(), @@ -186,24 +187,24 @@ describe('SshRelaySession abandoned remote PTYs', () => { expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID) }) - it('retires the lease without a kill when the relay proves the PTY is gone', async () => { - // pty.attach verifies process liveness before answering not-found, so this is the one branch - // with positive proof of death — and a dead process needs no shutdown request. + // INVERTED for STA-3077 step E-0. This case previously asserted that a bare not-found retires the + // lease and reports `pty:exit { code: -1 }`, on the premise that attach proves liveness before + // answering not-found. It does not: the relay answers not-found for a pane-identity mismatch and + // for any id it merely cannot hand back, so the premise licensed a fabricated death certificate. + // The clause is kept — inverted — so the new intent stays covered rather than silently dropped. + it('leaves the shell running when the relay only reports the PTY as not found', async () => { const { deps, shutdown } = await establishWithFailingReattach( new Error('PTY "pty-live" not found') ) expect(shutdown).not.toHaveBeenCalled() - expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( 'target-1', 'pty-live', 'expired' ) - expect(clearProviderPtyState).toHaveBeenCalledWith(APP_PTY_ID) - expect(deps.mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { - id: APP_PTY_ID, - code: -1 - }) + expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID) + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) }) it('keeps a recovered session attached when reattach succeeds after an earlier drop', async () => { diff --git a/src/main/ssh/ssh-relay-reattach-exit-proof.test.ts b/src/main/ssh/ssh-relay-reattach-exit-proof.test.ts new file mode 100644 index 00000000000..dc4bae2e8d2 --- /dev/null +++ b/src/main/ssh/ssh-relay-reattach-exit-proof.test.ts @@ -0,0 +1,358 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { formatPtyExitedError } from '../../shared/ssh-pty-failure-tokens' +import { SshRelaySession } from './ssh-relay-session' +import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' + +const { acceptOutputExitMock, muxRequestMock } = vi.hoisted(() => ({ + acceptOutputExitMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn() +})) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: vi.fn( + async (_mux: unknown, options: { clientInstanceId: string }) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + }) + ) +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: vi.fn().mockResolvedValue(undefined), + acceptSshPtyOutputExit: acceptOutputExitMock, + allocateSshPtyProviderGeneration: vi.fn(() => 17), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + applySshPtySourceRecoveryCancellationProof: vi.fn(() => true), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}) +})) +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) +vi.mock('./ssh-remote-orca-cli', () => ({ + runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }) +})) +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = muxRequestMock + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = vi.fn() + isDisposed = vi.fn().mockReturnValue(false) + } +})) +vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({ + installRemoteManagedAgentHooks: vi.fn().mockResolvedValue([]) +})) +vi.mock('../providers/ssh-pty-provider', () => ({ + SshPtyProvider: class MockSshPtyProvider { + onData = vi.fn().mockReturnValue(() => {}) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockReturnValue(() => {}) + attach = vi.fn().mockResolvedValue(undefined) + attachForReconnect = vi.fn().mockResolvedValue({}) + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../ipc/pty', () => ({ + resolvePaneShellTabId: vi.fn(() => undefined), + registerSshPtyProvider: vi.fn(), + unregisterSshPtyProvider: vi.fn(), + getSshPtyProvider: vi.fn(), + getPtyIdsForConnection: vi.fn().mockReturnValue([]), + clearPtyOwnershipForConnection: vi.fn(), + clearProviderPtyState: vi.fn(), + deletePtyOwnership: vi.fn(), + setPtyOwnership: vi.fn(), + restorePtyIncarnation: vi.fn(), + isCurrentPtyExit: vi.fn(() => true), + answerStartupTerminalColorQueriesForPty: vi.fn((_id: string, data: string) => data) +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { + registerSshPtyProvider, + getSshPtyProvider, + getPtyIdsForConnection, + clearProviderPtyState, + deletePtyOwnership +} = await import('../ipc/pty') + +const TARGET_ID = 'target-1' +const RELAY_PTY_ID = 'pty-live' +const APP_PTY_ID = 'ssh:target-1@@pty-live' +const LEAF_ID = '11111111-1111-4111-8111-111111111111' +const PROVIDER_GENERATION = 17 + +// The relay's own not-found wording; `isSshPtyNotFoundError` matches it and nothing narrows it +// further, so it arrives carrying no evidence about whether the shell is alive. +const UNPROVEN_NOT_FOUND = `PTY "${RELAY_PTY_ID}" not found` + +function detachedLease() { + return { + targetId: TARGET_ID, + ptyId: RELAY_PTY_ID, + state: 'detached' as const, + worktreeId: 'worktree-1', + tabId: 'tab-1', + leafId: LEAF_ID, + incarnationId: 'inc-proven-exit' + } +} + +/** + * STA-3077 / design E-0: a reattach that ends in a bare not-found tells us the relay cannot hand + * this id back — never that the process died. Synthesizing `pty:exit { code: -1 }` from it is a + * fabricated death certificate: the renderer treats it as a real exit and the surviving remote + * shell (and any agent inside it) is written off. Only an exit the relay actually observed may + * report an exit; everything else routes into the non-destructive detached branch. + */ +describe('SshRelaySession reattach failures never fabricate an exit', () => { + let warn: ReturnType + + beforeEach(() => { + vi.clearAllMocks() + muxRequestMock.mockReset() + muxRequestMock.mockResolvedValue([]) + acceptOutputExitMock.mockResolvedValue(undefined) + mockDeploySuccess() + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + vi.spyOn(console, 'info').mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + /** Drives the real connect-path reattach loop, so the failure reaches the handler as production does. */ + async function reattachFailsWith(message: string): Promise<{ + deps: ReturnType + attachForReconnect: ReturnType + }> { + const deps = createMockDeps() + const attachForReconnect = vi.fn().mockRejectedValue(new Error(message)) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect, + shutdown: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([detachedLease()] as ReturnType< + typeof deps.mockStore.getSshRemotePtyLeases + >) + const session = new SshRelaySession( + TARGET_ID, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + + await session.establish(deps.mockConn) + // Precondition, not a claim: if the reattach never ran, every "did not destroy" clause below + // would pass for the wrong reason. + expect(attachForReconnect).toHaveBeenCalled() + + return { deps, attachForReconnect } + } + + function warnings(): string[] { + return warn.mock.calls.map((call) => String(call[0])) + } + + it('reaches the failure handler through the real reattach path', async () => { + // Producer pin: without this the remaining clauses could pass on a path production never takes. + const { attachForReconnect } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(attachForReconnect.mock.calls[0]?.[0]).toBe(RELAY_PTY_ID) + }) + + it('sends no synthetic pty:exit to the pane on an unproven not-found', async () => { + const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + }) + + it('keeps pty ownership on an unproven not-found', async () => { + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deletePtyOwnership).not.toHaveBeenCalledWith(APP_PTY_ID) + }) + + it('keeps provider pty state on an unproven not-found', async () => { + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID) + }) + + it('leaves the lease terminable instead of expiring it on an unproven not-found', async () => { + const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + TARGET_ID, + RELAY_PTY_ID, + 'expired' + ) + }) + + // The counterpart. An exit the relay watched settles it, so the record is retired rather than + // re-attached on every future reconnect for the life of the install. `terminated`, not `expired`: + // expiry is the state the recovery grant reads, and retiring a record must not also authorise a + // replacement. + it('retires the lease on an exit the relay proved', async () => { + const { deps } = await reattachFailsWith( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-proven-exit') + ) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledWith(TARGET_ID, [ + RELAY_PTY_ID + ]) + }) + + it('persists a reconnect batch of proven exits in one async write', async () => { + const deps = createMockDeps() + const secondPtyId = 'pty-second' + const secondIncarnationId = 'inc-proven-second-exit' + const attachForReconnect = vi.fn().mockImplementation((ptyId: string) => { + const incarnationId = ptyId === RELAY_PTY_ID ? 'inc-proven-exit' : secondIncarnationId + return Promise.reject(new Error(formatPtyExitedError(ptyId, 0, incarnationId))) + }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect, + shutdown: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([ + detachedLease(), + { + ...detachedLease(), + ptyId: secondPtyId, + leafId: '22222222-2222-4222-8222-222222222222', + incarnationId: secondIncarnationId + } + ] as ReturnType) + const session = new SshRelaySession( + TARGET_ID, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + + await session.establish(deps.mockConn) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledOnce() + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledWith(TARGET_ID, [ + RELAY_PTY_ID, + secondPtyId + ]) + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalled() + }) + + it('does not retire the lease for proof naming another PTY', async () => { + const { deps } = await reattachFailsWith( + formatPtyExitedError('pty-some-other-shell', 0, 'inc-proven-exit') + ) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled() + }) + + it('does not retire the lease for proof naming another incarnation', async () => { + const { deps } = await reattachFailsWith( + formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-some-other-shell') + ) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled() + }) + + it('does not retire the lease when the failure proves nothing', async () => { + const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled() + }) + + it('routes an unproven not-found into the non-destructive detached branch', async () => { + // That branch announces itself with the "leaving detached" warning; the destructive one says + // "dropping stale". The log is how the two are told apart from outside the session. + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(warnings().some((line) => line.includes(RELAY_PTY_ID) && /detached/i.test(line))).toBe( + true + ) + }) + + it('does not take the drop-stale branch on an unproven not-found', async () => { + await reattachFailsWith(UNPROVEN_NOT_FOUND) + + expect(warnings().some((line) => /Dropping stale PTY/i.test(line))).toBe(false) + }) + + it('still reports an exit the relay actually observed', async () => { + // Contrast: the only evidence that authorizes a death claim is an exit the relay saw on a live + // stream. Removing the fabricated one must not mute this one. + const deps = createMockDeps() + vi.mocked(getSshPtyProvider).mockReturnValue({ + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue( + [] as ReturnType + ) + const session = new SshRelaySession( + TARGET_ID, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + await session.establish(deps.mockConn) + const provider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { + onExit: ReturnType + } + const onExit = provider.onExit.mock.calls[0]?.[0] as (payload: unknown) => void + + onExit({ + id: APP_PTY_ID, + code: 3, + incarnationId: 'incarnation-1', + providerGeneration: PROVIDER_GENERATION, + ptyIncarnation: 'incarnation-1' + }) + + await vi.waitFor(() => + expect(acceptOutputExitMock).toHaveBeenCalledWith({ + id: APP_PTY_ID, + code: 3, + providerGeneration: PROVIDER_GENERATION, + ptyIncarnation: 'incarnation-1' + }) + ) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( + TARGET_ID, + RELAY_PTY_ID, + 'terminated' + ) + expect(deletePtyOwnership).toHaveBeenCalledWith(APP_PTY_ID) + }) +}) diff --git a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts index f54afef3f89..17952f4ade0 100644 --- a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts +++ b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts @@ -156,6 +156,7 @@ function createSession(targetId: string): InstanceType { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), diff --git a/src/main/ssh/ssh-relay-session-data-delivery.test.ts b/src/main/ssh/ssh-relay-session-data-delivery.test.ts index 2eca6802f28..2a6edf1840e 100644 --- a/src/main/ssh/ssh-relay-session-data-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-data-delivery.test.ts @@ -403,7 +403,6 @@ describe('SshRelaySession data delivery', () => { expect(retryCalls[0]).toHaveProperty('resume') expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, Object.freeze({ status: 'checkpointUnavailable' }) ) second.dispose() @@ -762,7 +761,6 @@ describe('SshRelaySession data delivery', () => { expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, expect.objectContaining({ status: 'checkpoint', deliveryToken: 'old-token', diff --git a/src/main/ssh/ssh-relay-session-model-migration.test.ts b/src/main/ssh/ssh-relay-session-model-migration.test.ts index ae5db55e0cc..ab1512d82dc 100644 --- a/src/main/ssh/ssh-relay-session-model-migration.test.ts +++ b/src/main/ssh/ssh-relay-session-model-migration.test.ts @@ -199,7 +199,6 @@ describe('SshRelaySession model migration', () => { expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, Object.freeze({ status: 'checkpoint', clientGeneration: 1, @@ -259,7 +258,6 @@ describe('SshRelaySession model migration', () => { expect(attachForReconnectMock).toHaveBeenCalledWith( 'pty-1', - undefined, Object.freeze({ status: 'checkpointUnavailable' }) ) }) diff --git a/src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts b/src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts new file mode 100644 index 00000000000..5c52aac8ea9 --- /dev/null +++ b/src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts @@ -0,0 +1,334 @@ +/** + * The superseded-PTY fence must be LIVE on the relay reattach path — the path it + * was built for. Production wiring only: the real `pty:spawn`/`pty:write` + * handlers from src/main/ipc/pty.ts and a real `SshRelaySession` whose reconnect + * drives `restoreReattachedPtyRuntime`; only the wire below the provider is fake. + * + * Today reattach binds the pane through `runtime.registerPty` while spawn binds + * through `rememberPaneKeyForPty`, so the fence maps never learn the successor + * and a keystroke queued for the superseded PTY still reaches the host. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { makePaneKey } from '../../shared/stable-pane-id' + +const handlers = new Map unknown>() +const listeners = new Map unknown>() + +vi.mock('electron', () => ({ + BrowserWindow: undefined, + app: { + isPackaged: true, + getPath: () => '/tmp/orca-reattach-fence-test', + getVersion: () => '0.0.0-test' + }, + powerMonitor: { on: vi.fn() }, + nativeTheme: { shouldUseDarkColors: true }, + ipcMain: { + handle: (channel: string, handler: (event: unknown, args: unknown) => unknown) => { + handlers.set(channel, handler) + }, + on: (channel: string, listener: (event: unknown, args: unknown) => unknown) => { + listeners.set(channel, listener) + }, + removeHandler: (channel: string) => handlers.delete(channel), + removeAllListeners: (channel: string) => listeners.delete(channel) + } +})) + +const { providerInstances, spawnedPtyIds, attachIncarnationId } = vi.hoisted(() => ({ + providerInstances: [] as unknown[], + spawnedPtyIds: [] as string[], + attachIncarnationId: 'incarnation-reattached' +})) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) +vi.mock('./ssh-remote-orca-cli', () => ({ + runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }) +})) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: vi.fn( + async (_mux: unknown, options: { clientInstanceId: string }) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + }) + ) +})) +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = vi.fn().mockResolvedValue([]) + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = vi.fn() + isDisposed = vi.fn().mockReturnValue(false) + } +})) +vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({ + installRemoteManagedAgentHooks: vi.fn() +})) +// The relay session constructs and registers this provider itself, so the real +// pty.ts registry hands the same instance to pty:spawn and pty:write. +vi.mock('../providers/ssh-pty-provider', () => ({ + isSshPtyNotFoundError: (error: unknown) => String(error).includes('not found'), + isSshPtyIdentityMismatchError: (error: unknown) => String(error).includes('identity mismatch'), + SshPtyProvider: class MockSshPtyProvider { + spawn = vi.fn(async () => ({ id: spawnedPtyIds.shift() ?? 'ssh:target-1@@pty-unexpected' })) + write = vi.fn() + resize = vi.fn() + shutdown = vi.fn() + sendSignal = vi.fn() + getCwd = vi.fn() + getInitialCwd = vi.fn() + clearBuffer = vi.fn() + acknowledgeDataEvent = vi.fn() + hasChildProcesses = vi.fn() + getForegroundProcess = vi.fn() + serialize = vi.fn() + revive = vi.fn() + listProcesses = vi.fn(async () => []) + getDefaultShell = vi.fn() + getProfiles = vi.fn() + onData = vi.fn().mockReturnValue(() => {}) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockReturnValue(() => {}) + attach = vi.fn().mockResolvedValue(undefined) + attachForReconnect = vi.fn().mockResolvedValue({ incarnationId: attachIncarnationId }) + dispose = vi.fn() + + constructor() { + providerInstances.push(this) + } + } +})) +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { SshRelaySession } = await import('./ssh-relay-session') +const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') +const { + registerPtyHandlers, + unregisterSshPtyProvider, + getPtyIdForPaneKey, + getPtyIdsForConnection, + deletePtyOwnership +} = await import('../ipc/pty') + +const TARGET = 'target-1' +const TAB_ID = 'tab-1' + +type ProviderMock = { write: ReturnType } + +function makeMainWindow() { + const webContents = { + id: 1, + send: vi.fn(), + on: vi.fn(), + once: vi.fn(), + removeListener: vi.fn(), + isDestroyed: () => false + } + return { isDestroyed: () => false, isVisible: () => true, isMinimized: () => false, webContents } +} + +/** A live layout in which `leafId` sits in `tabId` — the tab the pane is in RIGHT NOW. + * The tab must also exist in `tabsByWorktree`: resolution ignores a layout whose tab is gone, so + * a fixture without one would resolve nothing and the clause would pass for the wrong reason. */ +function sessionWithLeafInTab(tabId: string, leafId: string) { + return { + tabsByWorktree: { 'worktree-1': [{ id: tabId, worktreeId: 'worktree-1' }] }, + terminalLayoutsByTabId: { + [tabId]: { root: { type: 'leaf' as const, leafId }, ptyIdsByLeafId: {} } + } + } +} + +function makeStore(leases: unknown[], session?: ReturnType) { + return { + ...(session ? { getWorkspaceSession: vi.fn(() => session) } : {}), + getRepos: vi.fn().mockReturnValue([]), + getSshPtyConsumerRecovery: vi.fn().mockReturnValue(null), + upsertSshPtyConsumerRecovery: vi.fn(), + removeSshPtyConsumerRecovery: vi.fn(), + getSshRemotePtyLeases: vi.fn(() => leases), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), + markSshRemotePtyLease: vi.fn(), + markSshRemotePtyLeases: vi.fn(), + markSshRemotePtyLeasesAsync: vi.fn(), + markSshRemotePtyLeasesForShutdown: vi.fn(), + markSshRemotePtyLeasesAttachedAsync: vi.fn(), + upsertSshRemotePtyLease: vi.fn(), + persistPtyBinding: vi.fn().mockReturnValue(true) + } +} + +/** + * One pane: spawned over SSH, then rebound by a reconnect whose durable lease + * names a different remote PTY — the STA-3077 shape. + */ +async function spawnThenReattachPane(leafId: string, currentTabId?: string) { + const mainWindow = makeMainWindow() + const predecessorPtyId = `ssh:${TARGET}@@pty-old-${leafId.slice(0, 4)}` + const reattachedRelayPtyId = `pty-new-${leafId.slice(0, 4)}` + const leases: unknown[] = [] + const store = makeStore( + leases, + currentTabId ? sessionWithLeafInTab(currentTabId, leafId) : undefined + ) + const runtime = { onPtySpawned: vi.fn(), registerPty: vi.fn() } + const session = new SshRelaySession( + TARGET, + vi.fn().mockReturnValue(mainWindow) as never, + store as never, + { removeAllForwards: vi.fn() } as never, + runtime as never + ) + + registerPtyHandlers( + mainWindow as never, + undefined, + undefined, + undefined, + undefined, + store as never + ) + await session.establish({} as never) + + spawnedPtyIds.push(predecessorPtyId) + await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + env: { ORCA_PANE_KEY: makePaneKey(TAB_ID, leafId) }, + connectionId: TARGET, + worktreeId: 'worktree-1', + tabId: TAB_ID, + leafId + }) + + // The relay restarted; the pane's durable lease now names a different shell. + leases.push({ + targetId: TARGET, + ptyId: reattachedRelayPtyId, + state: 'detached' as const, + worktreeId: 'worktree-1', + tabId: TAB_ID, + leafId + }) + await session.reconnect({} as never) + // Reconnect builds a fresh provider; writes route through the live one. + const provider = providerInstances.at(-1) as ProviderMock + + const write = (id: string): void => { + listeners.get('pty:write')!({ sender: mainWindow.webContents }, { id, data: 'x' }) + } + return { + runtime, + paneKey: makePaneKey(currentTabId ?? TAB_ID, leafId), + predecessorPtyId, + reattachedPtyId: `ssh:${TARGET}@@${reattachedRelayPtyId}`, + provider, + write, + dispose: () => { + deletePtyOwnership(predecessorPtyId) + deletePtyOwnership(`ssh:${TARGET}@@${reattachedRelayPtyId}`) + unregisterSshPtyProvider(TARGET) + } + } +} + +describe('a relay reattach binds the pane through the same producer as spawn', () => { + let cleanup: (() => void) | undefined + + beforeEach(() => { + handlers.clear() + listeners.clear() + providerInstances.length = 0 + spawnedPtyIds.length = 0 + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, + platform: 'linux-x64' + } as never) + }) + + afterEach(() => { + cleanup?.() + cleanup = undefined + }) + + it('refuses a queued write to the PTY the reattach superseded', async () => { + const pane = await spawnThenReattachPane('11111111-1111-4111-8111-111111111111') + cleanup = pane.dispose + pane.provider.write.mockClear() + // Still an owned, routable PTY — so only the fence can refuse the write. + expect(getPtyIdsForConnection(TARGET)).toContain(pane.predecessorPtyId) + + // A keystroke the renderer queued before the reconnect, still addressed to + // the shell the pane no longer owns. + pane.write(pane.predecessorPtyId) + await Promise.resolve() + + expect(pane.provider.write).not.toHaveBeenCalled() + }) + + it('resolves the pane to the reattached PTY', async () => { + const pane = await spawnThenReattachPane('22222222-2222-4222-8222-222222222222') + cleanup = pane.dispose + + expect(getPtyIdForPaneKey(pane.paneKey)).toBe(pane.reattachedPtyId) + }) + + // The lease's tabId is frozen at write time, but `detachTerminalPaneToTab` moves a live pane + // into a new tab without killing its PTY. A producer that forwarded `lease.tabId` would key the + // fence to the tab the pane LEFT — and every other clause here uses one tab on both sides, so + // nothing else in this file would notice. + it('keys the pane to the tab the leaf is in now, not the one its lease was written in', async () => { + const leafId = '44444444-4444-4444-8444-444444444444' + const pane = await spawnThenReattachPane(leafId, 'tab-moved-to') + cleanup = pane.dispose + + expect(pane.paneKey).toBe(makePaneKey('tab-moved-to', leafId)) + expect(getPtyIdForPaneKey(pane.paneKey)).toBe(pane.reattachedPtyId) + expect(getPtyIdForPaneKey(makePaneKey(TAB_ID, leafId))).not.toBe(pane.reattachedPtyId) + // The runtime graph must land on the SAME tab as the record and the fence. Registering under + // the lease's frozen tabId would split the pane across two tabs and ensure a mobile surface + // for the one it left. + expect(pane.runtime.registerPty).toHaveBeenCalledWith( + pane.reattachedPtyId, + 'worktree-1', + TARGET, + expect.objectContaining({ tabId: 'tab-moved-to', leafId }) + ) + }) + + // Guards the clause above from a fence that simply refuses everything. + it('still delivers a write to the reattached PTY', async () => { + const pane = await spawnThenReattachPane('33333333-3333-4333-8333-333333333333') + cleanup = pane.dispose + pane.provider.write.mockClear() + + pane.write(pane.reattachedPtyId) + await Promise.resolve() + + expect(pane.provider.write).toHaveBeenCalledWith(pane.reattachedPtyId, 'x') + }) +}) diff --git a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts index 3eebb3574b3..b958c26db1b 100644 --- a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts @@ -101,6 +101,9 @@ vi.mock('../providers/ssh-git-provider', () => ({ SshGitProvider: class MockSshGitProvider {} })) vi.mock('../ipc/pty', () => ({ + resolvePaneShellTabId: vi.fn(() => undefined), + // Step F: the single pane->shell bind producer the relay now calls. + bindPaneShell: vi.fn(() => true), registerSshPtyProvider: vi.fn(), unregisterSshPtyProvider: vi.fn(), getSshPtyProvider: vi.fn(), @@ -124,6 +127,7 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ })) const { + bindPaneShell, registerSshPtyProvider, getSshPtyProvider, getPtyIdsForConnection, @@ -435,18 +439,43 @@ describe('SshRelaySession reconnect incarnation ordering', () => { incarnationId }) expect(runtime.onPtySpawned).not.toHaveBeenCalled() - expect(mockStore.persistPtyBinding).toHaveBeenCalledWith({ + // Step F moved the observation point to the one bind producer; the intent is + // unchanged — exact incarnation proof, and mayCreate:false so reattach binds + // an existing pane and never grafts one back. + expect(bindPaneShell).toHaveBeenCalledWith({ + store: expect.anything(), worktreeId: 'worktree-1', tabId: 'tab-1', leafId: INCARNATION_LEAF_ID, ptyId: APP_PTY_ID, - incarnationId + incarnationId, + mayCreate: false }) - expect(vi.mocked(mockStore.persistPtyBinding).mock.invocationCallOrder[0]).toBeLessThan( + expect(vi.mocked(bindPaneShell).mock.invocationCallOrder[0]).toBeLessThan( vi.mocked(mockStore.markSshRemotePtyLeasesAttachedAsync).mock.invocationCallOrder[0]! ) }) + it('sends pane identity beside incarnation for relays that only understand the legacy fence', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const attachForReconnect = vi.fn().mockResolvedValue({ incarnationId: 'incarnation-lease' }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([ + { ...detachedLease(), incarnationId: 'incarnation-lease' } + ] as ReturnType) + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + + await session.establish(mockConn) + + expect(attachForReconnect).toHaveBeenCalledWith('pty-live', undefined, 'incarnation-lease', { + paneKey: `tab-1:${INCARNATION_LEAF_ID}`, + tabId: 'tab-1' + }) + }) + it('does not restore a PTY whose matching exit shares the attach reply batch', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const incarnationId = 'incarnation-exited-during-attach' @@ -489,7 +518,8 @@ describe('SshRelaySession reconnect incarnation ordering', () => { expect(runtime.registerPty).not.toHaveBeenCalled() expect(restorePtyIncarnation).toHaveBeenCalledWith(APP_PTY_ID, incarnationId) expect(setPtyOwnership).not.toHaveBeenCalled() - expect(mockStore.persistPtyBinding).not.toHaveBeenCalled() + // Step F: observed at the one bind producer, else this negative is vacuous. + expect(bindPaneShell).not.toHaveBeenCalled() expect(sourceActivationLease.rollback).toHaveBeenCalledOnce() expect(sourceActivationLease.commit).not.toHaveBeenCalled() expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( @@ -548,8 +578,14 @@ describe('SshRelaySession reconnect incarnation ordering', () => { incarnationId: currentIncarnationId }) expect(setPtyOwnership).toHaveBeenCalledWith(APP_PTY_ID, 'target-1') - expect(mockStore.persistPtyBinding).toHaveBeenCalledWith( - expect.objectContaining({ ptyId: APP_PTY_ID, incarnationId: currentIncarnationId }) + // Step F moved the observation point to the one bind producer; the asserted + // intent — exact incarnation proof, mayCreate:false — is unchanged. + expect(bindPaneShell).toHaveBeenCalledWith( + expect.objectContaining({ + ptyId: APP_PTY_ID, + incarnationId: currentIncarnationId, + mayCreate: false + }) ) expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:replay', { id: APP_PTY_ID, @@ -567,7 +603,9 @@ describe('SshRelaySession reconnect incarnation ordering', () => { vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([detachedLease()] as ReturnType< typeof mockStore.getSshRemotePtyLeases >) - vi.mocked(mockStore.persistPtyBinding).mockImplementationOnce(() => { + // Step F: the durable write now happens inside the bind producer, so that is + // where the failure has to be injected. + vi.mocked(bindPaneShell).mockImplementationOnce(() => { throw new Error('disk full') }) const runtime = { onPtySpawned: vi.fn(), registerPty: vi.fn() } diff --git a/src/main/ssh/ssh-relay-session-recovery-races.test.ts b/src/main/ssh/ssh-relay-session-recovery-races.test.ts index 26dd798ac5a..f8853832ed1 100644 --- a/src/main/ssh/ssh-relay-session-recovery-races.test.ts +++ b/src/main/ssh/ssh-relay-session-recovery-races.test.ts @@ -705,7 +705,7 @@ describe('SshRelaySession recovery race fencing', () => { await session.reconnect(deps.mockConn) expect(attachForReconnectMock).toHaveBeenCalledTimes(2) - expect(attachForReconnectMock.mock.calls.at(-1)?.[2]).toMatchObject({ + expect(attachForReconnectMock.mock.calls.at(-1)?.[1]).toMatchObject({ status: 'checkpoint', deliveryToken: 'new-token', acceptedSourceEndSu: 8 @@ -763,7 +763,7 @@ describe('SshRelaySession recovery race fencing', () => { const replacementReconnect = session.reconnect(deps.mockConn) await Promise.all([staleReconnect, replacementReconnect]) - const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[2]) + const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[1]) expect(recoveryRequests).toHaveLength(2) expect(recoveryRequests[1]).toMatchObject({ status: 'checkpoint', diff --git a/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts b/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts index e0f1572d4d3..9e83768ca42 100644 --- a/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-rejected-delivery.test.ts @@ -54,6 +54,7 @@ type RejectedDeliverySession = { > rejectedPtyRecoveryAttempts: Map sourceIdentityByRelayPtyId: Map + watchMuxForRelayLoss: (mux: SshChannelMultiplexer) => void retireExitedPty: (payload: { id: string code: number @@ -89,9 +90,18 @@ function rejectedPayload(overrides: Partial = {}): SshPtyData function prepareSession() { const deps = createMockDeps() + const disposeHandlers: ((reason: 'shutdown' | 'connection_lost') => void)[] = [] const mux = { isDisposed: vi.fn(() => false), - dispose: vi.fn() + onDispose: vi.fn((handler: (reason: 'shutdown' | 'connection_lost') => void) => { + disposeHandlers.push(handler) + return () => {} + }), + dispose: vi.fn((reason: 'shutdown' | 'connection_lost' = 'shutdown') => { + for (const handler of disposeHandlers) { + handler(reason) + } + }) } as unknown as SshChannelMultiplexer const session = new SshRelaySession( 'target-1', @@ -101,6 +111,11 @@ function prepareSession() { ) const internals = session as unknown as RejectedDeliverySession internals.mux = mux + // Why wired: disposing this mux is what fires the host-wide relay-lost reconnect, so a test can + // observe the blast radius of a per-PTY escalation instead of only the local dispose call. + const relayLost = vi.fn() + session.setOnRelayLost(relayLost) + internals.watchMuxForRelayLoss(mux) internals.activePtyProviderGeneration = 23 internals.ptyConsumerSessionState = { mode: 'negotiated', @@ -110,7 +125,7 @@ function prepareSession() { ownerLease: 'owner-lease', outputFlowControl: { version: 1, windowSu: 64 } } - return { deps, internals, mux, session } + return { deps, internals, mux, relayLost, session } } describe('SshRelaySession rejected PTY delivery recovery', () => { @@ -231,7 +246,8 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { const recovery = reattachKnownPty.mock.calls[0]?.[0] expect(recovery?.ptyId).toBe('pty-bad') expect(Array.from(recovery?.activeLeaseByPtyId.keys() ?? [])).toEqual(['pty-bad']) - expect(Array.from(recovery?.expectedIdentityByPtyId.keys() ?? [])).toEqual([]) + // The sibling `expectedIdentityByPtyId` clause is gone with the map itself (STA-3077 step A): + // reattach no longer carries pane identity at all, which subsumes "carries none for this pty". expect(recovery?.targetedDeliveryRecovery).toBe('fresh-activation') expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [ 'pty-bad' @@ -243,8 +259,10 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { expect(mux.dispose).not.toHaveBeenCalled() }) + // Why this one still drops the channel: the relay refused to confirm the delivery was canceled, so + // no per-PTY fence is available — it is an unprovable state, not an exhausted retry count. it('reconnects instead of canceling an unprovable malformed delivery', async () => { - const { internals, mux } = prepareSession() + const { internals, mux, relayLost } = prepareSession() const reattach = vi.fn().mockResolvedValue(true) internals.reattachRejectedPty = reattach vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -259,6 +277,7 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { ) expect(mux.dispose).toHaveBeenCalledWith('connection_lost') + expect(relayLost).toHaveBeenCalledWith('target-1') expect(reattach).not.toHaveBeenCalled() }) @@ -338,7 +357,8 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { internals.reattachRejectedPty('pty-bad', mux, 23, 'confirm-existing') ).resolves.toBe(true) - expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', undefined, checkpoint) + // Two args since step A dropped the expected-identity parameter from the attach call. + expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', checkpoint) expect(commit).toHaveBeenCalledOnce() expect(mux.dispose).not.toHaveBeenCalled() }) @@ -357,11 +377,11 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { expect(internals.rejectedPtyRecoveryAttempts).toHaveLength(0) }) - // Why a channel drop rather than a terminal relay error: a terminal error clears the reconnect - // backoff and rotates provider authority, aborting every fs and git request on the target, so one - // PTY's undeliverable output would strand the whole connection in manual recovery. - it('bounds failed targeted recovery and escalates to a recoverable relay reconnect', async () => { - const { internals, mux, session } = prepareSession() + // Why this is the oracle: an exhausted retry budget is not proof of anything, and the relay channel + // is shared — dropping it rotates provider authority, aborts every in-flight fs and git request on + // the target and stalls every sibling PTY over one PTY's undeliverable output. + it('parks one PTY after its recovery budget runs out and leaves the shared channel and a sibling alone', async () => { + const { internals, mux, relayLost, session } = prepareSession() const reattach = vi.fn().mockResolvedValue(false) internals.reattachRejectedPty = reattach getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider) @@ -370,19 +390,90 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) await internals.acceptPtyData(rejectedPayload()) - await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 }) - - expect(reattach).toHaveBeenCalledTimes(2) - expect(reattach.mock.calls).toEqual([ - ['pty-bad', mux, 23, 'confirm-existing'], - ['pty-bad', mux, 23, 'confirm-existing'] - ]) - expect(mux.dispose).toHaveBeenCalledWith('connection_lost') - expect(warn).toHaveBeenCalledWith( - expect.stringContaining('PTY pty-bad delivery recovery exhausted') + await vi.waitFor( + () => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('PTY pty-bad delivery recovery exhausted') + ), + { timeout: 10_000 } ) + // Why the extra wait: a dispose would be scheduled off the same recovery pass as the warning. + await new Promise((resolve) => setTimeout(resolve, 400)) + + expect(mux.dispose).not.toHaveBeenCalled() + expect(relayLost).not.toHaveBeenCalled() expect(onTerminalError).not.toHaveBeenCalled() + expect(reattach).toHaveBeenCalledTimes(12) expect(acceptOutputDataMock).not.toHaveBeenCalled() + + await internals.acceptPtyData( + rejectedPayload({ + id: 'ssh:target-1@@pty-healthy', + data: 'healthy', + ptyIncarnation: 'incarnation-healthy', + source: source({ + relayPtyId: 'pty-healthy', + spanId: 'token-healthy:0:7', + deliveryToken: 'token-healthy', + sourceEndSu: 7 + }), + sourceRejected: undefined + }) + ) + + expect(acceptOutputDataMock).toHaveBeenCalledOnce() + expect(acceptOutputDataMock).toHaveBeenCalledWith( + expect.objectContaining({ id: 'ssh:target-1@@pty-healthy', data: 'healthy' }) + ) + }) + + // The clause above pins the containment; this one pins that containment is not abandonment. The + // park's only stated escape is the next relay open, and a shared channel that stays healthy never + // gives it one — so a pane could sit with no output and no way back, which is the exact state + // this change exists to prevent. The park has to expire. + it('re-arms a parked PTY once its cooldown has passed rather than leaving it dark', async () => { + const { internals } = prepareSession() + const reattach = vi.fn().mockResolvedValue(false) + internals.reattachRejectedPty = reattach + getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await internals.acceptPtyData(rejectedPayload()) + await vi.waitFor( + () => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('PTY pty-bad delivery recovery exhausted') + ), + { timeout: 10_000 } + ) + expect(reattach).toHaveBeenCalledTimes(12) + + // Let the exhaustion loop go quiet first: its last retries are timer-driven and would + // otherwise land after the clear below and satisfy the final clause on their own. + await new Promise((resolve) => setTimeout(resolve, 1500)) + reattach.mockClear() + + // Every frame below reuses the ORIGINAL delivery token, because that is what actually happens: + // retirement is ours, not the host's, and a stalled source keeps publishing the token we + // retired. It also matters that they are identical — retirement holds ONE key per relay PTY, + // so slipping a different token in here would overwrite that key, un-retire the original and + // let the final clause pass with the wake-up path still broken. + + // Before the cooldown: parked, so nothing retries. + await internals.acceptPtyData(rejectedPayload()) + await new Promise((resolve) => setTimeout(resolve, 600)) + expect(reattach).not.toHaveBeenCalled() + + // Backdate the park instead of burning a minute of real time — the property under test is + // elapsed time, not which timer implementation measures it. + const parked = internals.rejectedPtyRecoveryAttempts.get('ssh:target-1@@pty-bad') as { + parkedAt: number + } + parked.parkedAt -= 60_000 + + await internals.acceptPtyData(rejectedPayload()) + + await vi.waitFor(() => expect(reattach).toHaveBeenCalled(), { timeout: 10_000 }) }) it('stops without an error when the rejected PTY exited during recovery', async () => { @@ -404,6 +495,30 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { expect(mux.dispose).not.toHaveBeenCalled() }) + // The counterpart to the clause above, and the distinction the whole design rests on. hasPty is + // three-state: null means the provider has not listed the host yet, which is precisely the state + // a fresh provider is in right after the reconnect that produces rejected frames. Reading that + // as death drops the attempt and schedules nothing, and the delivery token is already retired, + // so nothing can revive it — the pane stays dark. Only an explicit false is proof. + it('keeps retrying while the PTY liveness is merely unknown', async () => { + const { internals, mux, session } = prepareSession() + const reattach = vi.fn().mockResolvedValue(false) + internals.reattachRejectedPty = reattach + getSshPtyProviderMock.mockReturnValue({ hasPty: () => null } as unknown as SshPtyProvider) + const onTerminalError = vi.fn() + session.setOnTerminalRelayError(onTerminalError) + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await internals.acceptPtyData(rejectedPayload()) + + // More than one attempt is the whole point: unknown liveness must not end recovery. + await vi.waitFor(() => expect(reattach.mock.calls.length).toBeGreaterThan(1), { + timeout: 10_000 + }) + expect(onTerminalError).not.toHaveBeenCalled() + expect(mux.dispose).not.toHaveBeenCalled() + }) + // Why bounded: the bulk reconnect path caps the identical attach work at 8, and a relay that // starts rejecting across every PTY at once would otherwise open one attach round trip per PTY. it('caps concurrent targeted reattaches and coalesces repeats for one PTY', async () => { @@ -452,18 +567,18 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { }) it('does not let accepted frames refill the recovery budget indefinitely', async () => { - const { internals, mux, session } = prepareSession() + const { internals, mux, relayLost, session } = prepareSession() const reattach = vi.fn().mockResolvedValue(true) internals.reattachRejectedPty = reattach getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider) const onTerminalError = vi.fn() session.setOnTerminalRelayError(onTerminalError) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) // Why alternating, with a fresh bad token each round: every rejection retires its own delivery, // so a flapping PTY only keeps asking for recovery by moving onto new ones, and the accepted - // frame in between is what used to clear the budget outright. Each reattach here succeeds, so - // the consecutive budget is cleared legitimately too — only the per-generation ceiling can stop - // it. + // frame in between is what used to clear the budget outright — only the per-generation ceiling + // can stop it. for (let round = 0; round < 40; round++) { await internals.acceptPtyData( rejectedPayload({ @@ -487,9 +602,17 @@ describe('SshRelaySession rejected PTY delivery recovery', () => { ) await Promise.resolve() } - await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 }) + await vi.waitFor( + () => + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('PTY pty-bad delivery recovery exhausted') + ), + { timeout: 2000 } + ) expect(onTerminalError).not.toHaveBeenCalled() + expect(mux.dispose).not.toHaveBeenCalled() + expect(relayLost).not.toHaveBeenCalled() expect(acceptOutputDataMock).toHaveBeenCalledTimes(40) expect(reattach.mock.calls.length).toBeLessThanOrEqual(12) }) diff --git a/src/main/ssh/ssh-relay-session-terminal-error.test.ts b/src/main/ssh/ssh-relay-session-terminal-error.test.ts index a0f855543ef..026f64aafeb 100644 --- a/src/main/ssh/ssh-relay-session-terminal-error.test.ts +++ b/src/main/ssh/ssh-relay-session-terminal-error.test.ts @@ -104,6 +104,7 @@ function createMockDeps(): { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), diff --git a/src/main/ssh/ssh-relay-session-test-fixtures.ts b/src/main/ssh/ssh-relay-session-test-fixtures.ts index c5f084a8785..6fd72afd5a5 100644 --- a/src/main/ssh/ssh-relay-session-test-fixtures.ts +++ b/src/main/ssh/ssh-relay-session-test-fixtures.ts @@ -21,12 +21,14 @@ export function createMockDeps(): SshRelaySessionTestDeps { upsertSshPtyConsumerRecovery: vi.fn(), removeSshPtyConsumerRecovery: vi.fn(), getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), markSshRemotePtyLease: vi.fn(), markSshRemotePtyLeases: vi.fn(), markSshRemotePtyLeasesAsync: vi.fn(), markSshRemotePtyLeasesForShutdown: vi.fn(), markSshRemotePtyLeasesAttachedAsync: vi.fn(), - persistPtyBinding: vi.fn() + markSshRemotePtyLeasesTerminatedAsync: vi.fn(), + persistPtyBinding: vi.fn().mockReturnValue(true) } as unknown as Store const mockPortForward = { removeAllForwards: vi.fn() diff --git a/src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts b/src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts new file mode 100644 index 00000000000..6ff4cf9f18d --- /dev/null +++ b/src/main/ssh/ssh-relay-session-two-host-delivery-isolation.integration.test.ts @@ -0,0 +1,402 @@ +/** + * Journey 4 — two distinct final-host connections live at the same time, and one + * host's delivery failure may not reach the other host or a sibling PTY. + * + * Everything below SSH is real: two `SshRelaySession`s, two real + * `SshChannelMultiplexer`s, two real `SshPtyProvider`s, and two real + * `RelayDispatcher`s speaking the relay wire over an in-process pipe. The fault + * is injected the way a desynced relay actually produces it — a `pty.data` + * frame whose source header does not parse, with a delivery identity the host + * cannot cancel — so the whole rejection/recovery machinery runs for real. + * + * The oracle is the blast radius of exhausting ONE PTY's delivery-recovery + * budget: the shared relay channel, the sibling PTY on the same host, and the + * entire second host must be untouched. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { Store } from '../persistence' +import type { SshPortForwardManager } from './ssh-port-forward' +import type { SshConnection } from './ssh-connection' +import type { MultiplexerTransport } from './ssh-channel-multiplexer' +import { RelayDispatcher } from '../../relay/dispatcher' +import { getSshPtyProvider, unregisterSshPtyProvider } from '../ipc/pty' +import type { SshPtyProvider } from '../providers/ssh-pty-provider' +import { toAppSshPtyId } from '../providers/ssh-pty-id' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import { + installSshPtyOutputIntake, + type acceptSshPtyOutputData +} from '../ipc/ssh-pty-output-intake-registry' +import type { SshPtyOutputIntake } from '../ipc/ssh-pty-output-intake' + +vi.mock('./ssh-relay-deploy', () => ({ + deployAndLaunchRelay: vi.fn() +})) + +const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') +const { SshRelaySession } = await import('./ssh-relay-session') + +type DeliveredFrame = Parameters[0] + +/** The relay-side half of one host: a real dispatcher plus wire-level controls. */ +type FakeRelayHost = { + transport: MultiplexerTransport + dispatcher: RelayDispatcher + /** Publishes output on the PTY's live delivery, advancing its source cursor. */ + publish: (relayPtyId: string, data: string) => void + /** + * Publishes a frame whose source header does not parse (`sourceLengthSu` + * disagrees with the payload) but which still carries a complete delivery + * identity this client never installed. That combination is what routes the + * rejection into the bounded per-PTY recovery budget rather than an immediate + * channel reconnect. + */ + publishMalformed: (relayPtyId: string, data: string) => void + requestCounts: Map + /** When true, `pty.attach` is refused — the relay cannot re-prove the delivery. */ + refuseAttach: { value: boolean } + dispose: () => void +} + +function createFakeRelayHost(): FakeRelayHost { + const clientDataCallbacks: ((data: Buffer) => void)[] = [] + const clientCloseCallbacks: (() => void)[] = [] + const requestCounts = new Map() + const refuseAttach = { value: false } + const deliveries = new Map< + string, + { deliveryToken: string; ptyIncarnation: string; sentSu: number } + >() + let spawnCount = 0 + + const transport: MultiplexerTransport = { + write: (data) => { + setImmediate(() => dispatcher.feed(data)) + }, + onData: (cb) => { + clientDataCallbacks.push(cb) + }, + onClose: (cb) => { + clientCloseCallbacks.push(cb) + }, + close: () => { + for (const cb of clientCloseCallbacks) { + cb() + } + } + } + + const dispatcher = new RelayDispatcher((data) => { + setImmediate(() => { + for (const cb of clientDataCallbacks) { + cb(data) + } + }) + }) + + const count = (method: string): void => { + requestCounts.set(method, (requestCounts.get(method) ?? 0) + 1) + } + + dispatcher.onRequest('pty.openClient', async (params) => { + count('pty.openClient') + return { + protocolVersion: 1, + serverBuildId: 'test-relay-build', + clientGeneration: 1, + role: 'session-owner', + ownerGeneration: 1, + ownerLease: 'test-owner-lease', + resumed: params.resume !== undefined, + capabilities: { + outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } + } + } + }) + dispatcher.onRequest('session.resolveHome', async (params) => ({ + resolvedPath: params.path === '~' ? '/home/orca' : params.path + })) + dispatcher.onRequest('git.listWorktrees', async () => []) + dispatcher.onRequest('ports.detect', async () => ({ ports: [], platform: 'linux' })) + dispatcher.onRequest('pty.spawn', async () => { + count('pty.spawn') + spawnCount += 1 + const id = `remote-pty-${spawnCount}` + const ptyIncarnation = `incarnation-${id}` + const deliveryToken = `delivery-${id}` + deliveries.set(id, { deliveryToken, ptyIncarnation, sentSu: 0 }) + return { + id, + incarnationId: ptyIncarnation, + sourceActivation: { + status: 'pending', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation, + deliveryToken, + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + } + } + }) + dispatcher.onRequest('pty.getCwd', async () => { + count('pty.getCwd') + return '/home/orca/project' + }) + // Why `canceled: false`: the host cannot retire a delivery identity it never + // issued. That is the branch that asks the client to confirm the existing + // delivery rather than to drop the channel, so recovery enters the bounded + // retry budget this test is about. + dispatcher.onRequest('pty.cancelDelivery', async () => { + count('pty.cancelDelivery') + return { canceled: false } + }) + dispatcher.onRequest('pty.attach', async () => { + count('pty.attach') + if (refuseAttach.value) { + // Why not a not-found error: not-found is positive proof the PTY is gone + // and takes the destructive path. A refusal is the "unknown" case, which + // is what keeps the recovery budget burning down. + throw new Error('relay_attach_refused') + } + return { id: 'unused' } + }) + + const delivery = (relayPtyId: string) => { + const state = deliveries.get(relayPtyId) + if (!state) { + throw new Error(`relay has no delivery for ${relayPtyId}`) + } + return state + } + + return { + transport, + dispatcher, + publish: (relayPtyId, data) => { + const state = delivery(relayPtyId) + state.sentSu += data.length + dispatcher.notify('pty.data', { + id: relayPtyId, + data, + deliveryToken: state.deliveryToken, + ptyIncarnation: state.ptyIncarnation, + clientGeneration: 1, + ownerGeneration: 1, + sourceEndSu: state.sentSu, + sourceLengthSu: data.length + }) + }, + publishMalformed: (relayPtyId, data) => { + dispatcher.notify('pty.data', { + id: relayPtyId, + data, + deliveryToken: 'delivery-this-client-never-installed', + ptyIncarnation: 'incarnation-unknown', + clientGeneration: 1, + ownerGeneration: 1, + sourceEndSu: data.length + 1, + sourceLengthSu: data.length + 1 + }) + }, + requestCounts, + refuseAttach, + dispose: () => dispatcher.dispose() + } +} + +function createStore(): Store { + return { + getRepos: vi.fn().mockReturnValue([]), + getSshPtyConsumerRecovery: vi.fn().mockReturnValue(null), + upsertSshPtyConsumerRecovery: vi.fn(), + removeSshPtyConsumerRecovery: vi.fn(), + getSshRemotePtyLeases: vi.fn().mockReturnValue([]), + supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0), + markSshRemotePtyLease: vi.fn(), + markSshRemotePtyLeases: vi.fn(), + markSshRemotePtyLeasesAsync: vi.fn(), + markSshRemotePtyLeasesForShutdown: vi.fn(), + markSshRemotePtyLeasesAttachedAsync: vi.fn(), + persistPtyBinding: vi.fn().mockReturnValue(true) + } as unknown as Store +} + +type LiveHost = { + targetId: string + relay: FakeRelayHost + session: InstanceType + provider: SshPtyProvider + relayLost: ReturnType + terminalError: ReturnType +} + +async function openHost(targetId: string): Promise { + const relay = createFakeRelayHost() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: relay.transport, + serverBuildId: 'test-relay-build', + platform: 'linux-x64' + }) + const session = new SshRelaySession( + targetId, + vi.fn().mockReturnValue({ + isDestroyed: () => false, + isVisible: () => true, + isMinimized: () => false, + webContents: { send: vi.fn() } + }), + createStore(), + { removeAllForwards: vi.fn().mockResolvedValue(undefined) } as unknown as SshPortForwardManager + ) + const relayLost = vi.fn() + const terminalError = vi.fn() + session.setOnRelayLost(relayLost) + session.setOnTerminalRelayError(terminalError) + await session.establish({} as SshConnection) + const provider = getSshPtyProvider(targetId) as SshPtyProvider | undefined + if (!provider) { + throw new Error(`no PTY provider registered for ${targetId}`) + } + return { targetId, relay, session, provider, relayLost, terminalError } +} + +describe('two live SSH hosts over real relay dispatchers', () => { + const hosts: LiveHost[] = [] + let delivered: DeliveredFrame[] = [] + let uninstallIntake: (() => void) | null = null + let warnSpy: ReturnType + + beforeEach(() => { + vi.clearAllMocks() + delivered = [] + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + // Why the real registration seam: the session publishes accepted output + // through `acceptSshPtyOutputData`, so installing a recording intake + // observes the production delivery boundary without stubbing the module. + uninstallIntake = installSshPtyOutputIntake({ + acceptData: async (event) => { + delivered.push(event) + return { status: 'accepted' } as never + }, + acceptExit: async () => {}, + closeGeneration: () => {}, + dispose: () => {} + } as unknown as SshPtyOutputIntake) + }) + + afterEach(() => { + for (const host of hosts.splice(0)) { + host.session.dispose() + host.relay.dispose() + unregisterSshPtyProvider(host.targetId) + } + uninstallIntake?.() + uninstallIntake = null + warnSpy.mockRestore() + }) + + /** + * Brings both hosts up, gives host A two PTYs and host B one, proves all three + * deliver, then burns host A's PTY-1 delivery-recovery budget to exhaustion. + * + * The two clauses are separate tests on purpose: a mutation that only breaks + * same-host containment must be seen to leave the cross-host clause green, + * which proves the clauses independently rather than jointly. + */ + async function exhaustOnePtyOnHostA(): Promise<{ + hostA: LiveHost + hostB: LiveHost + a2: { id: string } + b1: { id: string } + }> { + const hostA = await openHost('host-a') + const hostB = await openHost('host-b') + hosts.push(hostA, hostB) + + const a1 = await hostA.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' }) + const a2 = await hostA.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' }) + const b1 = await hostB.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' }) + expect(a1.id).toBe(toAppSshPtyId('host-a', 'remote-pty-1')) + expect(a2.id).toBe(toAppSshPtyId('host-a', 'remote-pty-2')) + expect(b1.id).toBe(toAppSshPtyId('host-b', 'remote-pty-1')) + // Two distinct final-host connections, each with its own provider identity. + expect(hostA.provider).not.toBe(hostB.provider) + + hostA.relay.publish('remote-pty-1', 'a1-before') + hostA.relay.publish('remote-pty-2', 'a2-before') + hostB.relay.publish('remote-pty-1', 'b1-before') + await vi.waitFor(() => expect(delivered).toHaveLength(3), { timeout: 5_000 }) + delivered = [] + + // The fault: one PTY on host A can never re-prove its delivery. + hostA.relay.refuseAttach.value = true + hostA.relay.publishMalformed('remote-pty-1', 'a1-poisoned') + await vi.waitFor( + () => + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('remote-pty-1 delivery recovery exhausted') + ), + { timeout: 60_000 } + ) + // Why the extra settle: a channel drop is scheduled off the same recovery + // pass that logs the exhaustion, not before it. + await new Promise((resolve) => setTimeout(resolve, 500)) + + // The budget was really spent on relay round trips, so the fault is live. + expect(hostA.relay.requestCounts.get('pty.attach') ?? 0).toBeGreaterThanOrEqual(12) + expect(hostA.relay.requestCounts.get('pty.cancelDelivery') ?? 0).toBeGreaterThan(0) + // The undeliverable frame itself never reached a consumer. + expect(delivered.some((frame) => frame.data === 'a1-poisoned')).toBe(false) + + return { hostA, hostB, a2, b1 } + } + + it('leaves the sibling PTY and the shared channel on the same host working', async () => { + const { hostA, a2 } = await exhaustOnePtyOnHostA() + + hostA.relay.publish('remote-pty-2', 'a2-after') + await vi.waitFor( + () => + expect( + delivered.filter((frame) => frame.id === a2.id && frame.data === 'a2-after'), + 'the sibling PTY stopped receiving output after its neighbour was parked' + ).toHaveLength(1), + { timeout: 10_000 } + ) + // A live request round trip: the shared channel still carries the fs/git + // control plane that dropping it would have aborted. + await expect(hostA.provider.getCwd(a2.id)).resolves.toBe('/home/orca/project') + expect(hostA.relayLost, 'host A lost its relay channel over one PTY').not.toHaveBeenCalled() + expect(hostA.terminalError, 'host A was parked in manual recovery').not.toHaveBeenCalled() + }, 120_000) + + it('leaves the second host entirely untouched', async () => { + const { hostB, b1 } = await exhaustOnePtyOnHostA() + + expect(hostB.relayLost, 'host B lost its relay channel').not.toHaveBeenCalled() + expect(hostB.terminalError, 'host B was parked in manual recovery').not.toHaveBeenCalled() + expect( + hostB.relay.requestCounts.get('pty.attach') ?? 0, + "host A's recovery reached host B's relay" + ).toBe(0) + expect( + hostB.relay.requestCounts.get('pty.cancelDelivery') ?? 0, + "host A's delivery cancellation reached host B's relay" + ).toBe(0) + expect( + hostB.relay.requestCounts.get('pty.openClient') ?? 0, + 'host B renegotiated its consumer session' + ).toBe(1) + await expect(hostB.provider.getCwd(b1.id)).resolves.toBe('/home/orca/project') + hostB.relay.publish('remote-pty-1', 'b1-after') + await vi.waitFor( + () => + expect( + delivered.filter((frame) => frame.id === b1.id && frame.data === 'b1-after') + ).toHaveLength(1), + { timeout: 10_000 } + ) + }, 120_000) +}) diff --git a/src/main/ssh/ssh-relay-session.test.ts b/src/main/ssh/ssh-relay-session.test.ts index 570fca21fd4..8563c552aaf 100644 --- a/src/main/ssh/ssh-relay-session.test.ts +++ b/src/main/ssh/ssh-relay-session.test.ts @@ -531,26 +531,8 @@ describe('SshRelaySession', () => { ) }) - it('forwards a lease tab identity to reattach so a reset relay cannot cross-wire it', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() - const { getSshPtyProvider } = await import('../ipc/pty') - const mockAttach = vi.fn().mockResolvedValue(undefined) - vi.mocked(getSshPtyProvider).mockReturnValue({ - attachForReconnect: mockAttach, - dispose: vi.fn() - } as unknown as ReturnType) - vi.mocked(getPtyIdsForConnection).mockReturnValue([]) - vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([ - { targetId: 'target-1', ptyId: 'pty-1', state: 'detached', tabId: 'tab-a' } - ] as ReturnType) - - const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) - await session.establish(mockConn) - - expect(mockAttach).toHaveBeenCalledWith('pty-1', { tabId: 'tab-a' }) - }) - - it('forwards a lease pane identity when leaf identity is available', async () => { + // New relays prioritize incarnation; old relays ignore that additive field and need this fence. + it('preserves pane identity for relays that predate incarnation fencing', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const { getSshPtyProvider } = await import('../ipc/pty') const mockAttach = vi.fn().mockResolvedValue(undefined) @@ -567,7 +549,40 @@ describe('SshRelaySession', () => { const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) - expect(mockAttach).toHaveBeenCalledWith('pty-1', { + expect(mockAttach).toHaveBeenCalledWith('pty-1', undefined, undefined, { + paneKey: `tab-a:${leafId}`, + tabId: 'tab-a' + }) + }) + + // This is the path that reattaches every known pty when a relay comes back — precisely when ids + // have been reissued from pty-1 — so it is the one that most needs to say which shell it means. + // The shell's own identity is authoritative; pane identity remains the fallback for old relays. + it('forwards the shell identity its lease recorded when reattaching', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const { getSshPtyProvider } = await import('../ipc/pty') + const mockAttach = vi.fn().mockResolvedValue(undefined) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + const leafId = '11111111-1111-4111-8111-111111111111' + vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([ + { + targetId: 'target-1', + ptyId: 'pty-1', + state: 'detached', + tabId: 'tab-a', + leafId, + incarnationId: 'inc-host-1' + } + ] as ReturnType) + + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + + expect(mockAttach).toHaveBeenCalledWith('pty-1', undefined, 'inc-host-1', { paneKey: `tab-a:${leafId}`, tabId: 'tab-a' }) @@ -602,13 +617,10 @@ describe('SshRelaySession', () => { await session.reconnect(mockConn) - expect(mockAttach).toHaveBeenCalledWith('pty-1', { - paneKey: `tab-old:${staleLeafId}`, - tabId: 'tab-old' - }) expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-1') expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-1') expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith('target-1', 'pty-1', 'expired') + expect(mockAttach).toHaveBeenCalledOnce() expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', { id: 'ssh:target-1@@pty-1', code: -1 @@ -671,7 +683,11 @@ describe('SshRelaySession', () => { expect(mockStore.markSshRemotePtyLeasesAttachedAsync).not.toHaveBeenCalled() }) - it('invalidates and broadcasts remote PTYs that cannot reattach after relay reconnect', async () => { + // INVERTED for STA-3077 step E-0. This case previously asserted that a PTY the relay reports as + // not-found is invalidated and broadcast as an exit. A not-found is not proof of death, so the + // broadcast was a fabricated one; what must still hold is that the healthy sibling reattaches and + // the unproven one is left alone rather than written off. + it('leaves an unreattachable remote PTY alone while its sibling reattaches', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) @@ -693,12 +709,9 @@ describe('SshRelaySession', () => { expect(mockAttach).toHaveBeenCalledWith('pty-stale') expect(mockAttach).toHaveBeenCalledWith('pty-live') - expect(clearProviderPtyState).toHaveBeenCalledWith('ssh:target-1@@pty-stale') - expect(deletePtyOwnership).toHaveBeenCalledWith('ssh:target-1@@pty-stale') - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { - id: 'ssh:target-1@@pty-stale', - code: -1 - }) + expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-stale') + expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-stale') + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) }) it('retries transient reattach failure without tearing down provider registration', async () => { diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index 820eb7e1360..ea56dd88786 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -12,6 +12,7 @@ import type { SshPtyAttachResult } from '../providers/ssh-pty-session-reattach' import type { SshPtyDataCallback, SshPtyExitCallback } from '../providers/ssh-pty-provider-contract' import type { SshPtyRecoveryActivationLease } from '../providers/ssh-pty-notification-routing' import { isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors' +import { parseMatchingPtyExitedError } from '../../shared/ssh-pty-failure-tokens' import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' import { SshFilesystemProvider } from '../providers/ssh-filesystem-provider' import { isMethodNotFoundError } from './ssh-filesystem-stream-reader' @@ -32,6 +33,8 @@ import { import { _internals as openCodeInternals } from '../opencode/hook-service' import { getPiAgentStatusExtensionSource } from '../pi/agent-status-extension-source' import { + bindPaneShell, + resolvePaneShellTabId, registerSshPtyProvider, unregisterSshPtyProvider, getSshPtyProvider, @@ -95,6 +98,8 @@ import { parseRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output' import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' +import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' +import { isValidTerminalTabId } from '../../shared/terminal-tab-id' import { SSH_AI_VAULT_LIST_SESSIONS_METHOD, SSH_AI_VAULT_LIST_SESSIONS_TIMEOUT_MS, @@ -103,8 +108,6 @@ import { type SshAiVaultRelayListParams, type SshAiVaultRelayTitleParams } from '../../shared/ssh-ai-vault-relay' -import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' -import { isValidTerminalTabId } from '../../shared/terminal-tab-id' import { openSshPtyConsumerSession, type OpenSshPtyConsumerSessionOptions, @@ -138,11 +141,15 @@ const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8 const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000 const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50 const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200 -const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2 -// Why a second ceiling: the consecutive budget resets whenever a reattach succeeds, so a PTY that -// alternates recovered and rejected frames would otherwise reattach forever — each one costs a -// store read, an attach round trip and a store write. +// Why a ceiling at all: a PTY that alternates recovered and rejected frames would otherwise reattach +// forever — each one costs a store read, an attach round trip and a store write. Exhausting it parks +// this PTY's delivery; it never authorizes tearing anything down. const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12 +// Why a park expires: exhaustion parks one PTY's delivery instead of dropping the shared channel, +// and the escape it names — the next relay open — may never come while that channel stays healthy. +// A pane with no output and no way back is the state this change exists to prevent, so the park is +// a cooldown rather than a verdict: recovery is rate-limited, never abandoned. +const SSH_REJECTED_PTY_RECOVERY_PARK_MS = 60_000 const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150 const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed' @@ -191,15 +198,15 @@ type RemoteCliBridgeEnv = { pathDelimiter?: ':' | ';' } -type ExpectedPtyIdentity = { paneKey?: string; tabId?: string } +type LegacyExpectedPtyIdentity = { paneKey?: string; tabId?: string } type TargetedDeliveryRecovery = 'confirm-existing' | 'fresh-activation' -function expectedIdentityForLease(lease: { +function legacyExpectedIdentityForLease(lease: { tabId?: string leafId?: string -}): ExpectedPtyIdentity | null { +}): LegacyExpectedPtyIdentity | undefined { if (typeof lease.tabId !== 'string' || lease.tabId.length === 0) { - return null + return undefined } const paneKey = isValidTerminalTabId(lease.tabId) && @@ -207,10 +214,7 @@ function expectedIdentityForLease(lease: { isTerminalLeafId(lease.leafId) ? makePaneKey(lease.tabId, lease.leafId) : undefined - return { - ...(paneKey ? { paneKey } : {}), - tabId: lease.tabId - } + return { ...(paneKey ? { paneKey } : {}), tabId: lease.tabId } } function parseRecoveryComplete(params: Record): PtySourceRecoveryComplete | null { @@ -343,9 +347,10 @@ export class SshRelaySession { string, { providerGeneration: number - attempts: number generationAttempts: number reported: boolean + /** When the budget ran out; 0 while it has not. Drives the park cooldown. */ + parkedAt: number } >() private readonly rejectedPtyRecoveryRetries = new Set>() @@ -1789,7 +1794,8 @@ export class SshRelaySession { const offeredSource = payload.source if ( offeredSource && - this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource) + this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource) && + !this.isParkedRecoveryDue(payload.id) ) { return Promise.resolve() } @@ -1846,6 +1852,19 @@ export class SshRelaySession { }) } + /** + * A parked PTY whose cooldown has elapsed must be let back past the retired-delivery filter. + * Retirement is ours, not the host's: a stalled source keeps publishing the very token we + * retired, so that filter would swallow every frame that could re-arm the park and the pane + * would stay dark for good — the failure the cooldown exists to rule out. Letting one through + * costs nothing: it is re-classified as rejected and re-retired, so no output reaches the + * terminal; it only regains the ability to ask for recovery. + */ + private isParkedRecoveryDue(appPtyId: string): boolean { + const attempt = this.rejectedPtyRecoveryAttempts.get(appPtyId) + return !!attempt?.parkedAt && Date.now() - attempt.parkedAt >= SSH_REJECTED_PTY_RECOVERY_PARK_MS + } + private recoverRejectedPtyDelivery( payload: SshPtyDataPayload, source: SshPtyDataPayload['source'] @@ -1880,26 +1899,28 @@ export class SshRelaySession { const attempt = previous?.providerGeneration === providerGeneration ? previous - : { providerGeneration, attempts: 0, generationAttempts: 0, reported: false } - if ( - attempt.attempts >= SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS || - attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS - ) { - if (!attempt.reported) { - attempt.reported = true - console.warn( - `[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; dropping the relay channel to reconnect` - ) - // Why a channel drop and not a terminal relay error: a terminal error clears the reconnect - // backoff, rotates provider authority (aborting every in-flight fs and git request on the - // target) and parks the target in a manual-recovery state — over one PTY's delivery. Losing - // the channel is the recoverable escalation, and it is what this path did before targeted - // recovery existed. - mux.dispose('connection_lost') + : { providerGeneration, generationAttempts: 0, reported: false, parkedAt: 0 } + if (attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS) { + // Why park instead of dropping the relay channel: a retry count is not proof of anything, and + // the channel is shared — dropping it rotates provider authority, aborts every in-flight fs + // and git request on the target and stalls every sibling PTY over one PTY's delivery. The + // remote shell keeps running and its lease stands. + if (attempt.parkedAt && Date.now() - attempt.parkedAt >= SSH_REJECTED_PTY_RECOVERY_PARK_MS) { + // The park has served its cooldown. Re-arm rather than leave the pane dark for good. + attempt.generationAttempts = 0 + attempt.reported = false + attempt.parkedAt = 0 + } else { + if (!attempt.reported) { + attempt.reported = true + attempt.parkedAt = Date.now() + console.warn( + `[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; parking its delivery for ${SSH_REJECTED_PTY_RECOVERY_PARK_MS}ms or until the next relay open` + ) + } + return } - return } - attempt.attempts++ attempt.generationAttempts++ this.rejectedPtyRecoveryAttempts.set(appPtyId, attempt) void this.rejectedPtyReattaches @@ -1915,13 +1936,9 @@ export class SshRelaySession { ) .then( (recovered) => { - if (recovered) { - // Why only a completed reattach clears this: an accepted frame proves nothing about the - // delivery that was rejected, and resetting on one lets a flapping PTY reattach forever. - attempt.attempts = 0 - return + if (!recovered) { + this.retryRejectedPtyDelivery(payload, source, appPtyId) } - this.retryRejectedPtyDelivery(payload, source, appPtyId) }, (error: unknown) => { console.warn(`[ssh-relay-session] PTY ${relayPtyId} targeted delivery recovery failed`, { @@ -1935,14 +1952,23 @@ export class SshRelaySession { // Why liveness is checked before retrying: reattachKnownPty resolves without claiming the lease // when the PTY exited mid-attach, which is indistinguishable from a failed reattach at the call - // site. Retrying that race twice would drop the relay channel over an ordinary PTY exit. + // site, so an ordinary exit would otherwise burn the whole recovery budget on attach round trips. private retryRejectedPtyDelivery( payload: SshPtyDataPayload, source: SshPtyDataPayload['source'], appPtyId: string ): void { const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined - if (!ptyProvider || typeof ptyProvider.hasPty !== 'function' || !ptyProvider.hasPty(appPtyId)) { + // Why `=== false` and not `!`: hasPty is three-state, and null means the provider has not + // listed the host yet — ignorance, not death, which is exactly the state a fresh provider is + // in right after the reconnect that produces these rejected frames. Treating it as death + // dropped the recovery attempt and scheduled nothing, and since the delivery token is already + // retired, no later frame could revive it: the pane stayed dark. Only proof stops us. + if ( + !ptyProvider || + typeof ptyProvider.hasPty !== 'function' || + ptyProvider.hasPty(appPtyId) === false + ) { this.rejectedPtyRecoveryAttempts.delete(appPtyId) return } @@ -1979,21 +2005,25 @@ export class SshRelaySession { const activeLeaseByPtyId = activeLease ? new Map([[relayPtyId, activeLease]]) : new Map() - const expectedIdentity = activeLease ? expectedIdentityForLease(activeLease) : undefined const attachedLeaseIds = new Set() + const terminatedLeaseIds = new Set() await this.reattachKnownPty({ ptyProvider, ptyId: relayPtyId, activeLeaseByPtyId, - expectedIdentityByPtyId: expectedIdentity - ? new Map([[relayPtyId, expectedIdentity]]) - : new Map(), attachedLeaseIds, + terminatedLeaseIds, mux, providerGeneration, shouldContinue, targetedDeliveryRecovery }) + if (terminatedLeaseIds.size > 0) { + await this.store.markSshRemotePtyLeasesTerminatedAsync( + this.targetId, + Array.from(terminatedLeaseIds) + ) + } if (attachedLeaseIds.size > 0 && shouldContinue()) { await this.store.markSshRemotePtyLeasesAttachedAsync( this.targetId, @@ -2245,21 +2275,21 @@ export class SshRelaySession { mux: SshChannelMultiplexer, shouldContinue: () => boolean ): Promise { + // Why: installs predating pane-keyed supersession carry duplicate live leases; + // fanning out over them is what grafted panes the user never opened (STA-3077). + const retired = await this.store.supersedeDuplicatePaneLeases(this.targetId) + if (retired > 0) { + console.info( + `[ssh-relay-session] Retired ${retired} duplicate pane lease(s) for ${this.targetId}; their remote shells are left running.` + ) + } const activeLeases = this.store .getSshRemotePtyLeases(this.targetId) .filter((lease) => lease.state !== 'terminated' && lease.state !== 'expired') const activeLeaseByPtyId = new Map(activeLeases.map((lease) => [lease.ptyId, lease])) const leasedPtyIds = activeLeases.map((lease) => lease.ptyId) - // Why: pass pane identity so the relay can reject cross-generation id collisions; tabId falls back for pre-leafId leases. - const expectedIdentityByPtyId = new Map( - activeLeases - .map((lease): [string, ExpectedPtyIdentity] | null => { - const expected = expectedIdentityForLease(lease) - return expected ? [lease.ptyId, expected] : null - }) - .filter((entry): entry is [string, ExpectedPtyIdentity] => entry !== null) - ) const attachedLeaseIds = new Set() + const terminatedLeaseIds = new Set() // Why: after app restart ptyOwnership is empty, but durable SSH leases still describe grace-window survivors. const ptyIds = Array.from( new Set([ @@ -2286,8 +2316,8 @@ export class SshRelaySession { ptyProvider, ptyId, activeLeaseByPtyId, - expectedIdentityByPtyId, attachedLeaseIds, + terminatedLeaseIds, mux, providerGeneration, shouldContinue @@ -2307,6 +2337,12 @@ export class SshRelaySession { await Promise.all( Array.from({ length: Math.min(SSH_PTY_REATTACH_MAX_CONCURRENCY, ptyIds.length) }, worker) ) + if (terminatedLeaseIds.size > 0) { + await this.store.markSshRemotePtyLeasesTerminatedAsync( + this.targetId, + Array.from(terminatedLeaseIds) + ) + } if (attachedLeaseIds.size > 0 && shouldContinue()) { await this.store.markSshRemotePtyLeasesAttachedAsync( this.targetId, @@ -2319,8 +2355,8 @@ export class SshRelaySession { ptyProvider: SshPtyProvider ptyId: string activeLeaseByPtyId: Map - expectedIdentityByPtyId: Map attachedLeaseIds: Set + terminatedLeaseIds: Set mux: SshChannelMultiplexer providerGeneration: number shouldContinue: () => boolean @@ -2330,8 +2366,8 @@ export class SshRelaySession { ptyProvider, ptyId, activeLeaseByPtyId, - expectedIdentityByPtyId, attachedLeaseIds, + terminatedLeaseIds, mux, providerGeneration, shouldContinue, @@ -2358,12 +2394,17 @@ export class SshRelaySession { targetedDeliveryRecovery === 'fresh-activation' ? undefined : await this.sourceRecoveryRequest(appPtyId) + // The shell this lease names, as the host attested it. Without it the main reconnect path + // attaches by id alone, and a replaced relay reissues ids from pty-1. const attachResult = await this.attachPtyWithRetry( ptyProvider, ptyId, - expectedIdentityByPtyId.get(ptyId), recoveryRequest, - shouldContinue + shouldContinue, + activeLeaseByPtyId.get(ptyId)?.incarnationId, + activeLeaseByPtyId.get(ptyId) + ? legacyExpectedIdentityForLease(activeLeaseByPtyId.get(ptyId)!) + : undefined ) sourceActivationLease = attachResult.sourceActivationLease if (!shouldContinue()) { @@ -2509,7 +2550,13 @@ export class SshRelaySession { if (!shouldContinue()) { return } - this.handlePtyReattachFailure(ptyId, appPtyId, pendingReattach, error) + this.handlePtyReattachFailure( + ptyId, + pendingReattach, + error, + activeLeaseByPtyId.get(ptyId)?.incarnationId, + terminatedLeaseIds + ) } finally { recoveryActivationLease?.retire() sourceActivationLease?.rollback() @@ -2548,22 +2595,42 @@ export class SshRelaySession { lease: SshPtyLease | undefined ): void { if (lease?.worktreeId && lease.tabId && lease.leafId) { - this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { - tabId: lease.tabId, - leafId: lease.leafId, - incarnationId - }) + // Why bind before registering: `mayCreate: false` refuses when the pane is + // gone, and registering first would surface a pane the user never opened. + // A thrown write is unknown, not a refusal, so it must not detach anything. + // The lease's tabId was frozen when it was written and the pane may have moved since, so the + // live layout outranks it. Resolved before the write, so a throw cannot lose it and leave the + // graph registered under the tab the pane left. + const tabId = resolvePaneShellTabId(this.store, lease.leafId) ?? lease.tabId + let bind: { bound: boolean; tabId: string } | null = null try { - this.store.persistPtyBinding({ + bind = bindPaneShell({ + store: this.store, worktreeId: lease.worktreeId, - tabId: lease.tabId, + tabId, leafId: lease.leafId, ptyId: appPtyId, - incarnationId + incarnationId, + mayCreate: false }) } catch (error) { console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error) } + if (bind?.bound === false) { + // Unresolved, not dead: the remote shell keeps running and stays + // reattachable once a durable pane names it again. + console.info( + `[ssh-relay-session] No durable pane owns ${appPtyId} on ${this.targetId}; left running unbound.` + ) + return + } + // The tab the bind resolved, not the lease's frozen one — a moved pane must not be recorded + // in the graph under the tab it left while its record and fence name the tab it is in. + this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { + tabId, + leafId: lease.leafId, + incarnationId + }) return } this.runtime?.onPtySpawned(appPtyId, incarnationId, { awaitsRegistration: false }) @@ -2572,9 +2639,10 @@ export class SshRelaySession { private async attachPtyWithRetry( ptyProvider: SshPtyProvider, ptyId: string, - expectedIdentity: ExpectedPtyIdentity | undefined, recoveryRequest: PtySourceRecoveryRequest | undefined, - shouldContinue: () => boolean + shouldContinue: () => boolean, + expectedIncarnationId: string | undefined, + legacyExpectedIdentity: LegacyExpectedPtyIdentity | undefined ): Promise { let lastError: unknown for (let attempt = 0; attempt < 2; attempt++) { @@ -2585,12 +2653,26 @@ export class SshRelaySession { return await this.attachPtyWithDeadline( ptyProvider, ptyId, - expectedIdentity, - recoveryRequest + recoveryRequest, + expectedIncarnationId, + legacyExpectedIdentity ) } catch (error) { lastError = error - if (!shouldContinue() || isSshPtyNotFoundError(error) || attempt === 1) { + // A proven exit is as final as an unknown id: retrying either only burns the wait before + // the same answer. Without this the liveness-probe reap, which now answers with proof + // rather than "not found", would no longer match and would cost a needless retry. + if ( + !shouldContinue() || + isSshPtyNotFoundError(error) || + isSshPtyIdentityMismatchError(error) || + parseMatchingPtyExitedError( + error instanceof Error ? error.message : String(error), + ptyId, + expectedIncarnationId + ) !== null || + attempt === 1 + ) { throw error } await this.waitForPtyReattachRetry() @@ -2602,8 +2684,9 @@ export class SshRelaySession { private async attachPtyWithDeadline( ptyProvider: SshPtyProvider, ptyId: string, - expectedIdentity: ExpectedPtyIdentity | undefined, - recoveryRequest: PtySourceRecoveryRequest | undefined + recoveryRequest: PtySourceRecoveryRequest | undefined, + expectedIncarnationId: string | undefined, + legacyExpectedIdentity: LegacyExpectedPtyIdentity | undefined ): Promise { let timer: ReturnType | undefined let timedOut = false @@ -2617,13 +2700,17 @@ export class SshRelaySession { timer.unref?.() }) try { - const attach = expectedIdentity - ? recoveryRequest - ? ptyProvider.attachForReconnect(ptyId, expectedIdentity, recoveryRequest) - : ptyProvider.attachForReconnect(ptyId, expectedIdentity) - : recoveryRequest - ? ptyProvider.attachForReconnect(ptyId, undefined, recoveryRequest) - : ptyProvider.attachForReconnect(ptyId) + const attach = + expectedIncarnationId !== undefined || legacyExpectedIdentity !== undefined + ? ptyProvider.attachForReconnect( + ptyId, + recoveryRequest, + expectedIncarnationId, + legacyExpectedIdentity + ) + : recoveryRequest !== undefined + ? ptyProvider.attachForReconnect(ptyId, recoveryRequest) + : ptyProvider.attachForReconnect(ptyId) const guardedAttach = attach.then((result) => { if (timedOut) { result.sourceActivationLease?.rollback() @@ -2650,40 +2737,31 @@ export class SshRelaySession { private handlePtyReattachFailure( ptyId: string, - appPtyId: string, pending: PendingPtyReattach, - error: unknown + error: unknown, + expectedIncarnationId: string | undefined, + terminatedLeaseIds: Set ): void { - if (!isSshPtyNotFoundError(error)) { - pending.restoreRequired = 'reattachAttemptsExhausted' - this.wakeRecovery(pending) - console.warn( - `[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after bounded reattach attempts failed: ${ - error instanceof Error ? error.message : String(error) - }` - ) - return - } - if (isSshPtyIdentityMismatchError(error)) { - console.warn( - `[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${ - error instanceof Error ? error.message : String(error) - }` - ) - return + // An exact exit proof settles the lease; queue it for one post-worker durable retirement batch. + // `terminated` is not the recovery-grant state, so this cannot authorize a replacement. + + const exitProof = parseMatchingPtyExitedError( + error instanceof Error ? error.message : String(error), + ptyId, + expectedIncarnationId + ) + if (exitProof) { + terminatedLeaseIds.add(ptyId) } + // Nothing else proves it — not even a not-found, which the relay also returns when it merely + // cannot hand this id back — so every other failure leaves the pane detached and recoverable. + pending.restoreRequired = 'reattachAttemptsExhausted' + this.wakeRecovery(pending) console.warn( - `[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${ + `[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after reattach failed: ${ error instanceof Error ? error.message : String(error) }` ) - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) - } } private async sourceRecoveryRequest( diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 0f90182d053..54e03a81f58 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -1469,6 +1469,8 @@ export type PreloadApi = { rows: number cwd?: string cwdFallback?: 'worktree' + /** Create a shell for this pane instead of adopting the one it records; see pty.ts. */ + createFreshShellForUnreachablePane?: boolean env?: Record envToDelete?: string[] command?: string diff --git a/src/preload/index.ts b/src/preload/index.ts index 58aec899f03..4808dd3b979 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -943,6 +943,7 @@ const api = { rows: number cwd?: string cwdFallback?: 'worktree' + createFreshShellForUnreachablePane?: boolean env?: Record envToDelete?: string[] command?: string diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 06eaa15d5ff..f45727c19db 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -48,7 +48,6 @@ import { IMMEDIATE_PTY_EXIT_TIMEOUT_MS, MAX_RELAY_PTY_SESSIONS, PtyHandler, - attachIdentityMismatches, formatNodePtyUnavailableMessage } from './pty-handler' import { RelayDispatcher } from './dispatcher' @@ -1386,10 +1385,10 @@ describe('PtyHandler', () => { } ) - it('reports not-found and reaps a reattach whose backing shell is dead', async () => { + it('proves the exit and reaps a reattach whose backing shell is dead', async () => { // Why: a lingering managed entry whose child died without an onExit would // otherwise attach-succeed with an empty replay and strand the pane on a - // black shell. A provably-dead pid must surface as not-found so the SSH + // black shell. A provably-dead pid must surface as a PROVEN exit so the SSH // provider maps it to SSH_SESSION_EXPIRED and the pane respawns fresh. let onExitCb: ((evt: { exitCode: number }) => void) | undefined mockPtySpawn.mockReturnValue({ @@ -1402,15 +1401,25 @@ describe('PtyHandler', () => { const exits: { id: string; paneKey?: string }[] = [] handler.setExitListener((evt) => exits.push(evt)) - await dispatcher.callRequest('pty.spawn', { env: { ORCA_PANE_KEY: 'tab-dead:0' } }) + const spawned = (await dispatcher.callRequest('pty.spawn', { + env: { ORCA_PANE_KEY: 'tab-dead:0' } + })) as { id: string; incarnationId: string } expect(handler.activePtyCount).toBe(1) expect(onExitCb).toBeDefined() const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) try { + // INVERTED: the reap is unchanged, but its answer is no longer the ordinary unknown. The + // relay held this shell and found its pid gone, which is first-hand proof of death — the + // one observation that separates a dead shell from one a replacement relay never knew. await expect( - dispatcher.callRequest('pty.attach', { id: 'pty-1', suppressReplayNotification: true }) - ).rejects.toThrow('PTY "pty-1" not found') + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) } finally { aliveSpy.mockRestore() } @@ -1419,9 +1428,16 @@ describe('PtyHandler', () => { // is freed so a later attach also cleanly reports not-found. expect(exits).toEqual([{ id: 'pty-1', paneKey: 'tab-dead:0' }]) expect(handler.activePtyCount).toBe(0) + // INVERTED: a LATER attach still answers, and now answers with what this process observed — + // the remembered exit outlives the reap, which is the whole point of remembering it. await expect( - dispatcher.callRequest('pty.attach', { id: 'pty-1', suppressReplayNotification: true }) - ).rejects.toThrow('PTY "pty-1" not found') + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) }) it.skipIf(process.platform === 'win32')( @@ -1476,7 +1492,10 @@ describe('PtyHandler', () => { onData: vi.fn(), onExit: vi.fn() }) - await dispatcher.callRequest('pty.spawn', {}) + const spawned = (await dispatcher.callRequest('pty.spawn', {})) as { + id: string + incarnationId: string + } let shutdown: Promise | undefined const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockImplementation(() => { @@ -1484,9 +1503,14 @@ describe('PtyHandler', () => { return false }) try { - await expect(dispatcher.callRequest('pty.attach', { id: 'pty-1' })).rejects.toThrow( - 'PTY "pty-1" not found' - ) + // INVERTED with the reap above: proof of exit, not an unknown id. + await expect( + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) await expect(shutdown).resolves.toBeUndefined() } finally { aliveSpy.mockRestore() @@ -1494,6 +1518,173 @@ describe('PtyHandler', () => { expect(mockKill).toHaveBeenCalledWith('SIGKILL') expect(handler.activePtyCount).toBe(0) + }) + + // A relay that never knew an id may be a replacement whose predecessor's shells are still + // running, so "not found" proves nothing. A relay that WATCHED the shell exit knows first-hand. + // Keeping that observation is what lets the client tell a dead shell from an unknown one, and + // therefore what lets it respawn without risking a second agent on a live transcript. + // What the host answers reaches clients that predate the reply, and one of those reads an + // unrecognized attach error as neither death nor recovery — a stranded pane. So the proof is + // offered only to a client that says it can act on it. + /** `toThrow(expect.not.stringMatching(...))` passes whenever ANYTHING throws, so a negative + * claim about the wording has to read the message itself. */ + async function attachRejectionMessage(params: Record): Promise { + const error = await dispatcher.callRequest('pty.attach', params).then( + () => null, + (thrown: unknown) => thrown as Error + ) + expect(error, 'attach unexpectedly succeeded').toBeInstanceOf(Error) + return (error as Error).message + } + + it('keeps the older wording for a client that has not declared it understands exit proof', async () => { + const spawned = await spawnPty({}) + const onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as + | ((evt: { exitCode: number }) => void) + | undefined + onExitCb?.({ exitCode: 3 }) + + await expect(dispatcher.callRequest('pty.attach', { id: spawned.id })).rejects.toThrow( + `PTY "${'pty-1'}" not found` + ) + }) + + // The sharpest edge in the whole change. A caller that names no shell is not asking about this + // one, so a remembered exit is not an answer to it: relay A is killed leaving pty-1 alive and + // orphaned, relay B mints its own pty-1 and THAT one exits. A pane carrying no recorded identity + // must not be told its shell is gone, because the reply authorizes replacing a process that is + // still running — the double agent resume this whole change exists to prevent. + it('does not prove an exit to a caller that names no shell', async () => { + const spawned = await spawnPty({}) + const onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as + | ((evt: { exitCode: number }) => void) + | undefined + onExitCb?.({ exitCode: 3 }) + + expect(await attachRejectionMessage({ id: spawned.id, exitProofSupported: true })).not.toMatch( + /SSH_PTY_EXITED/ + ) + }) + + // The same rule on the other proof route. A replacement relay can hold a DIFFERENT shell under + // this id; if that one dies, its death is not evidence about the caller's, which may be running + // orphaned under the relay this one replaced. The reap still happens — a dead shell should be + // cleared whoever asked — but the answer must not authorize replacing a live process. + it('does not prove a reaped shell to a caller that named a different one', async () => { + const spawned = await spawnPty({}) + const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) + try { + const message = await attachRejectionMessage({ + id: spawned.id, + exitProofSupported: true, + expectedIncarnationId: 'a-shell-from-a-previous-relay' + }) + expect(message).not.toMatch(/SSH_PTY_EXITED/) + expect(message).toMatch(/identity mismatch/i) + } finally { + aliveSpy.mockRestore() + } + // The reap still ran: the dead entry is gone regardless of who asked. + expect(handler.activePtyCount).toBe(0) + }) + + it('answers a later attach with the exit it observed, not an unknown id', async () => { + const spawned = await spawnPty({}) + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + // The exit listener was captured at spawn by the shared mock; drive a real exit through it. + onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as typeof onExitCb + expect(onExitCb).toBeDefined() + onExitCb!({ exitCode: 3 }) + + const asking = { + id: spawned.id, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + } + await expect(attachPty(asking)).rejects.toThrow(/SSH_PTY_EXITED/) + await expect(attachPty(asking)).rejects.toThrow(/code=3/) + }) + + // The dangerous shape: a replaced relay hands pty-1 to a new shell, that one exits, and the + // pane still bound to the ORIGINAL pty-1 asks about its own. Reporting the stranger's death as + // proof would authorize a respawn onto a shell that may still be running. + it('does not report a remembered exit for a shell the caller did not ask about', async () => { + const spawned = await spawnPty({}) + const onExitCb = mockPtyInstance.onExit.mock.calls.at(-1)?.[0] as + | ((evt: { exitCode: number }) => void) + | undefined + onExitCb?.({ exitCode: 3 }) + + expect( + await attachRejectionMessage({ + id: spawned.id, + exitProofSupported: true, + expectedIncarnationId: 'a-different-shell' + }) + ).not.toMatch(/SSH_PTY_EXITED/) + }) + + // INVERTED from the deleted `attachIdentityMismatches` suite. That guard compared the paneKey and + // tabId frozen at spawn; it caught a recycled id, but it also refused a pane that had merely moved + // to another tab — and refused as "not found", which the client read as death and answered by + // resuming the agent a second time. The property worth keeping is "a recycled id must not attach"; + // the property that had to go is "identity is where the pane lives". Both are pinned below, now + // against the shell's own incarnation. + it('refuses an attach whose expected incarnation names a different shell', async () => { + const first = await spawnPty({}) + + await expect( + dispatcher.callRequest('pty.attach', { + id: first.id, + expectedIncarnationId: 'incarnation-from-a-previous-relay' + }) + ).rejects.toThrow(/identity mismatch/i) + }) + + // The wording matters as much as the refusal: "not found" is what the client maps to an expired + // session, and expiry authorizes a respawn onto the very shell this branch proved is alive. + it('does not word an incarnation refusal as not-found', async () => { + const first = await spawnPty({}) + + expect( + await attachRejectionMessage({ + id: first.id, + expectedIncarnationId: 'incarnation-from-a-previous-relay' + }) + ).not.toMatch(/not found/i) + }) + + it('attaches when the expected incarnation is the shell it names', async () => { + const first = await spawnPty({}) + + await expect( + attachPty({ id: first.id, expectedIncarnationId: first.incarnationId }) + ).resolves.toMatchObject({ incarnationId: first.incarnationId }) + }) + + // An older client sends no expectation. Refusing there would strand every pane it owns. + it('stays permissive when the caller supplies no expected incarnation', async () => { + const first = await spawnPty({}) + + await expect(attachPty({ id: first.id })).resolves.toMatchObject({ + incarnationId: first.incarnationId + }) + }) + + // The regression the old tab-keyed guard caused. A pane carries its shell to a new tab; the + // relay still holds the paneKey/tabId frozen at spawn. That must not refuse the attach. + it('attaches a pane that has moved to another tab', async () => { + const first = await spawnPty({ paneKey: 'tab-a:0', tabId: 'tab-a' }) + + await expect( + attachPty({ + id: first.id, + expectedIncarnationId: first.incarnationId, + expectedPaneKey: 'tab-b:0', + expectedTabId: 'tab-b' + }) + ).resolves.toMatchObject({ incarnationId: first.incarnationId }) expect(vi.getTimerCount()).toBe(0) }) @@ -2202,13 +2393,17 @@ describe('PtyHandler', () => { expect(spawnOptions.env.ORCA_PANE_KEY).toBeUndefined() expect(spawnOptions.env.ORCA_TAB_ID).toBeUndefined() + // A caller that names no incarnation is an older client, and pane identity is the only fence + // it can be given — so it still applies, and still refuses. A caller that CAN name the shell is + // fenced on that instead and attaches regardless of which tab it now lives in; that is pinned + // by "attaches a pane that has moved to another tab". await expect( attachPty({ id: 'pty-1', expectedPaneKey: 'tab-b:leaf-b', expectedTabId: 'tab-b' }) - ).rejects.toThrow('PTY "pty-1" not found') + ).rejects.toThrow(/identity mismatch/i) await expect( attachPty({ @@ -3451,13 +3646,15 @@ describe('PtyHandler', () => { expect(callArgs.env.ORCA_PANE_KEY).toBeUndefined() expect(callArgs.env.ORCA_TAB_ID).toBeUndefined() + // Revive preserves the serialized pane identity, and for a caller that names no incarnation + // that identity is still the fence. await expect( dispatcher.callRequest('pty.attach', { id: 'pty-1', expectedPaneKey: 'tab-other:leaf', expectedTabId: 'tab-other' }) - ).rejects.toThrow('PTY "pty-1" not found') + ).rejects.toThrow(/identity mismatch/i) }) it('invokes the exit listener with the spawn-time paneKey', async () => { @@ -3769,13 +3966,19 @@ describe('PtyHandler', () => { mockPtySpawn.mockReturnValue({ ...mockPtyInstance, onData: vi.fn(), onExit: vi.fn() }) const poolEmpty = vi.fn() handler.onPtyPoolEmpty(poolEmpty) - await spawnPty() + const spawned = await spawnPty() const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) try { - await expect(attachPty({ id: 'pty-1', suppressReplayNotification: true })).rejects.toThrow( - 'PTY "pty-1" not found' - ) + // INVERTED with the reap above: proof of exit, not an unknown id. + await expect( + attachPty({ + id: 'pty-1', + suppressReplayNotification: true, + exitProofSupported: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow(/SSH_PTY_EXITED/) } finally { aliveSpy.mockRestore() } @@ -3928,39 +4131,3 @@ describe('PtyHandler', () => { expect(handler.pendingPtyCreationCount).toBe(0) }) }) - -describe('attachIdentityMismatches', () => { - it('rejects a paneKey collision across relay generations', () => { - // Old lease expects tab-a's pane; the reset relay's pty-1 belongs to tab-b. - expect( - attachIdentityMismatches({ paneKey: 'tab-a:0' }, { paneKey: 'tab-b:0', tabId: 'tab-b' }) - ).toBe(true) - }) - - it('rejects a tabId collision when only tab identity is known', () => { - expect(attachIdentityMismatches({ tabId: 'tab-a' }, { tabId: 'tab-b' })).toBe(true) - }) - - it('accepts a matching identity', () => { - expect( - attachIdentityMismatches( - { paneKey: 'tab-a:0', tabId: 'tab-a' }, - { paneKey: 'tab-a:0', tabId: 'tab-a' } - ) - ).toBe(false) - }) - - it('stays permissive when the caller supplies no identity', () => { - expect(attachIdentityMismatches({}, { paneKey: 'tab-a:0', tabId: 'tab-a' })).toBe(false) - }) - - it('stays permissive when the managed PTY predates identity capture', () => { - expect(attachIdentityMismatches({ paneKey: 'tab-a:0', tabId: 'tab-a' }, {})).toBe(false) - }) - - it('does not reject on tabId when paneKey matches (split panes share a tab)', () => { - expect( - attachIdentityMismatches({ paneKey: 'tab-a:1' }, { paneKey: 'tab-a:1', tabId: 'tab-a' }) - ).toBe(false) - }) -}) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 3f8bd593e08..b63b1cb631c 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -26,6 +26,7 @@ import { normalizeRuntimePathForComparison } from '../shared/cross-platform-path' import { splitWorktreeId } from '../shared/worktree-id' +import { formatPtyExitedError } from '../shared/ssh-pty-failure-tokens' import { PhysicalExitTracker } from '../shared/physical-exit-tracker' import { SHELL_READY_MARKER_PREFIX } from '../main/shell-ready-marker-scanner' import { @@ -176,6 +177,12 @@ type RelayAgentSessionCreateResult = { sourceActivation?: PtySourceReceivingActivation } +/** Enough to cover every pane a host could plausibly have lost since the client last looked. */ +const MAX_REMEMBERED_PTY_EXITS = 256 + +/** No exit status to report: the relay found the pid gone rather than watching it go. */ +const PTY_EXIT_CODE_OBSERVED_GONE = -1 + const AGENT_SESSION_CREATE_OPERATION_ID_PATTERN = /^[A-Za-z0-9_-]{43}$/ const AGENT_SESSION_CREATE_OPERATION_RETENTION_MS = 24 * 60 * 60 * 1000 const AGENT_SESSION_CREATE_OPERATION_LIMIT = 4_096 @@ -342,12 +349,11 @@ export type PtyExitListener = (event: { id: string; paneKey?: string }) => void type PtyIdentity = { paneKey?: string; tabId?: string } -/** - * True when a reattach's expected pane identity contradicts the target PTY's own. - * Rejects cross-relay-generation id collisions (a reset relay reuses `pty-N`). - * Only compares fields present on both sides; absent identity stays permissive. - */ -export function attachIdentityMismatches(expected: PtyIdentity, managed: PtyIdentity): boolean { +/** Fallback fence for a client too old to name a shell by incarnation. It froze paneKey/tabId at + * spawn, so it refuses a pane that merely moved tabs — which is why a client that CAN name the + * shell is fenced on that instead. Not deleted, because the relay is shared: an upgraded host + * would otherwise leave every not-yet-updated client attaching a recycled id unchecked. */ +function attachIdentityMismatches(expected: PtyIdentity, managed: PtyIdentity): boolean { return Boolean( (expected.paneKey && managed.paneKey && expected.paneKey !== managed.paneKey) || (expected.tabId && managed.tabId && expected.tabId !== managed.tabId) @@ -377,6 +383,10 @@ export class PtyHandler { private outputFlushTimer: ReturnType | null = null private pendingOutputByPty = new Map() private pendingExitByPty = new Map() + /** Exits this process WATCHED, kept past publication so attach can prove death rather than report + * an unknown id. Bounded and oldest-evicted; losing one costs a user-asked respawn, never a + * wrong one, and a crash losing all of them correctly reads as no knowledge. */ + private exitedPtys = new Map() private pausedOutputPtys = new Set() private consumerPausedOutputPtys = new Set() private removeLegacyCapacityListener: (() => void) | null = null @@ -800,6 +810,7 @@ export class PtyHandler { code: exitCode, incarnationId: managed.incarnationId }) + this.rememberPtyExit(managed.id, exitCode, managed.incarnationId) this.publishPendingExit(managed.id) this.notifyExitListener(managed) this.agentSessionOwners.release(managed.id) @@ -810,6 +821,29 @@ export class PtyHandler { }) } + /** Proof must name the caller's own shell: present and matching. Both gone-paths ask this, so a + * change to the rule cannot reach one and miss the other. */ + private exitProofAnswersCaller( + expectedIncarnationId: string | undefined, + exitedIncarnationId: string, + clientUnderstandsExitProof: boolean + ): boolean { + return clientUnderstandsExitProof && expectedIncarnationId === exitedIncarnationId + } + + /** Oldest-evicted: a Map iterates in insertion order, so the first key is the oldest. */ + private rememberPtyExit(id: string, code: number, incarnationId: string): void { + this.exitedPtys.delete(id) + this.exitedPtys.set(id, { code, incarnationId }) + while (this.exitedPtys.size > MAX_REMEMBERED_PTY_EXITS) { + const oldest = this.exitedPtys.keys().next() + if (oldest.done) { + break + } + this.exitedPtys.delete(oldest.value) + } + } + private releaseRelayIngress(managed: ManagedPty): void { const startupCommand = managed.startupCommand if (startupCommand) { @@ -1677,9 +1711,29 @@ export class PtyHandler { sourceActivation?: PtySourceReceivingActivation }> { const id = params.id as string + const expectedIncarnationId = + typeof params.expectedIncarnationId === 'string' ? params.expectedIncarnationId : undefined + // Gated because the host's answer reaches clients predating it, which read an unrecognized + // attach error as neither death nor recovery and strand the pane. + const clientUnderstandsExitProof = params.exitProofSupported === true const managed = this.ptys.get(id) // Why: after dispose, pty.kill is a POSIX no-op; treat disposed as not-found so failures aren't silent. if (!managed || managed.disposed) { + // `disposed` is teardown, not an observed exit, so only a genuinely absent pty may be + // answered from what this process watched exit. + if (!managed) { + const exited = this.exitedPtys.get(id) + if ( + exited && + this.exitProofAnswersCaller( + expectedIncarnationId, + exited.incarnationId, + clientUnderstandsExitProof + ) + ) { + throw new Error(formatPtyExitedError(id, exited.code, exited.incarnationId)) + } + } throw new Error(`PTY "${id}" not found`) } @@ -1693,19 +1747,43 @@ export class PtyHandler { disposeManagedPty(managed) this.removePty(id) this.clearPtyFlowState(id) - throw new Error(`PTY "${id}" not found`) + // The reap runs whoever asked, but the ANSWER depends on whose shell died: under this id a + // replacement relay may hold a different one entirely, and the caller's may be orphaned and + // alive under the relay this one replaced. + this.rememberPtyExit(id, PTY_EXIT_CODE_OBSERVED_GONE, managed.incarnationId) + if (expectedIncarnationId && expectedIncarnationId !== managed.incarnationId) { + throw new Error(`PTY "${id}" identity mismatch`) + } + throw new Error( + this.exitProofAnswersCaller( + expectedIncarnationId, + managed.incarnationId, + clientUnderstandsExitProof + ) + ? formatPtyExitedError(id, PTY_EXIT_CODE_OBSERVED_GONE, managed.incarnationId) + : `PTY "${id}" not found` + ) } - // Why: generation resets can reuse PTY IDs; reject conflicting identities. - const mismatch = attachIdentityMismatches( - { - paneKey: typeof params.expectedPaneKey === 'string' ? params.expectedPaneKey : undefined, - tabId: typeof params.expectedTabId === 'string' ? params.expectedTabId : undefined - }, - managed.attachIdentity ?? { paneKey: managed.paneKey, tabId: managed.tabId } - ) - if (mismatch) { - throw new Error(`PTY "${id}" not found (identity mismatch)`) + // A reset relay restarts ids at pty-1, so an id alone can name somebody else's shell. The + // incarnation is the shell's own identity, so it catches that without caring where the pane + // lives — unlike the pane identity below, which froze the tab at spawn and refused moved panes. + if ( + !expectedIncarnationId && + attachIdentityMismatches( + { + paneKey: typeof params.expectedPaneKey === 'string' ? params.expectedPaneKey : undefined, + tabId: typeof params.expectedTabId === 'string' ? params.expectedTabId : undefined + }, + managed.attachIdentity ?? { paneKey: managed.paneKey, tabId: managed.tabId } + ) + ) { + throw new Error(`PTY "${id}" identity mismatch`) + } + if (expectedIncarnationId && expectedIncarnationId !== managed.incarnationId) { + // Deliberately NOT worded "not found": that phrasing is what the client maps to an expired + // session, and expiry authorizes a respawn onto a shell this branch just proved is alive. + throw new Error(`PTY "${id}" identity mismatch`) } managed.startupIngress?.snapshotBarrier() diff --git a/src/renderer/src/components/terminal-pane/TerminalPane.tsx b/src/renderer/src/components/terminal-pane/TerminalPane.tsx index 323a3dc497e..adcc8541f1a 100644 --- a/src/renderer/src/components/terminal-pane/TerminalPane.tsx +++ b/src/renderer/src/components/terminal-pane/TerminalPane.tsx @@ -190,7 +190,7 @@ import { scheduleImagePasteWebglAtlasRecovery } from './terminal-webgl-atlas-rec import { restoreTerminalFitToDesktop, restoreTerminalFitsToDesktop } from './terminal-fit-restore' import { useVisibleTerminalTabClaim } from './use-visible-terminal-tab-claim' import { TerminalSshReconnectOverlay } from './TerminalSshReconnectOverlay' -import { TerminalRemoteRuntimeReconnectBanner } from './TerminalRemoteRuntimeReconnectBanner' +import { TerminalPaneDisconnectedBanner } from './TerminalPaneDisconnectedBanner' import { selectTerminalTabAgentTypesByLeaf } from './terminal-tab-agent-type-index' import { canContinueAgentSessionInNewSession } from './terminal-agent-session-continuation' import { @@ -2915,6 +2915,10 @@ function TerminalPane( resolveAgentForLeaf(contextMenuLeafId) ) // Each toggle gates on its own leaf (header=active, menu=opened-over), so mixed splits show it only where chat can render. + // The pane-level disconnected card owns the bottom strip while it is up; see the error toast below. + const activePaneShowsUnreachableCard = Boolean( + activePane && ptyRecoveryStatesByPaneId[activePane.id]?.unreachablePane + ) const activePaneCanToggleChat = canToggleChatForLeaf(activePane?.leafId ?? null) const contextMenuCanToggleChat = canToggleChatForLeaf(contextMenuLeafId) return ( @@ -2982,8 +2986,12 @@ function TerminalPane( ) })} {/* Why: the reconnect banner already owns SSH recovery UX; the z-50 error - toast was painting over it (same bottom strip) with the raw ssh:connect failure. */} - {terminalError && isActive && !showSshReconnectOverlay ? ( + toast was painting over it (same bottom strip) with the raw ssh:connect failure. + The pane's own disconnected card owns the same strip and the same story, and the toast + outranks it at z-50 — so it covered BOTH of its actions and made the affordance + unclickable. Naming only the connection overlay here was the gap: the card is raised for + a per-pane failure that leaves the connection up, which is exactly when this fires. */} + {terminalError && isActive && !showSshReconnectOverlay && !activePaneShowsUnreachableCard ? ( { + const unreachable = recoveryState.unreachablePane + if (unreachable) { + unreachable.onRetry() + return + } paneTransportsRef.current.get(pane.id)?.retryRecovery?.() }} + onStartNewTerminal={recoveryState.unreachablePane?.onStartNewTerminal} + onRestoreTerminalFocus={() => pane.terminal.focus()} />, pane.container, `remote-runtime-reconnect-${pane.id}` diff --git a/src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx b/src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx new file mode 100644 index 00000000000..a8229c9594e --- /dev/null +++ b/src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx @@ -0,0 +1,164 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' +import { cleanup, render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { TooltipProvider } from '@/components/ui/tooltip' +import { TerminalPaneDisconnectedBanner } from './TerminalPaneDisconnectedBanner' + +vi.mock('@/i18n/i18n', () => ({ + translate: (_key: string, fallback: string) => fallback +})) + +afterEach(cleanup) + +function renderBanner(banner: React.ReactNode): ReturnType { + return render({banner}) +} + +describe('TerminalPaneDisconnectedBanner', () => { + it('shows quiet bounded automatic recovery without a manual action', () => { + renderBanner( + + ) + + expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument() + expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument() + expect(screen.queryByRole('button')).not.toBeInTheDocument() + }) + + it('offers one explicit reconnect action after automatic recovery stops', async () => { + const onReconnect = vi.fn() + const user = userEvent.setup() + renderBanner( + + ) + + expect(screen.getByText('Remote runtime disconnected')).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Reconnect' })) + expect(onReconnect).toHaveBeenCalledOnce() + }) +}) + +/** + * STA-3077 step E-0. A reattach that fails without proving the shell is gone used to either + * respawn silently — resuming the same agent a second time into one transcript — or leave the pane + * frozen behind a raw error toast. It now renders here, with the two actions the design approved. + */ +describe('TerminalPaneDisconnectedBanner, unreachable SSH pane', () => { + function renderUnreachable() { + const onReconnect = vi.fn() + const onStartNewTerminal = vi.fn() + const onRestoreTerminalFocus = vi.fn() + renderBanner( + + ) + return { onReconnect, onStartNewTerminal, onRestoreTerminalFocus } + } + + it('offers exactly the two approved actions', async () => { + const user = userEvent.setup() + const { onReconnect, onStartNewTerminal } = renderUnreachable() + + const buttons = screen.getAllByRole('button') + expect(buttons.map((button) => button.textContent)).toEqual([ + 'Try again', + 'Start a new terminal' + ]) + + await user.click(screen.getByRole('button', { name: 'Try again' })) + expect(onReconnect).toHaveBeenCalledOnce() + await user.click(screen.getByRole('button', { name: 'Start a new terminal' })) + expect(onStartNewTerminal).toHaveBeenCalledOnce() + }) + + it('returns keyboard focus to this terminal after either action', async () => { + const user = userEvent.setup() + const onReconnect = vi.fn() + const onStartNewTerminal = vi.fn() + renderBanner( + <> +