diff --git a/tests/e2e/helpers/docker-ssh-relay-stall.ts b/tests/e2e/helpers/docker-ssh-relay-stall.ts new file mode 100644 index 00000000000..89e3e3eefba --- /dev/null +++ b/tests/e2e/helpers/docker-ssh-relay-stall.ts @@ -0,0 +1,70 @@ +/** + * Freezes the detached relay without killing it. + * + * Why this fault and not a transport sever: a sever reconnects through the + * checkpoint path and never classifies a reattach failure at all (see the header + * of ssh-maxsessions-remote-pid-binding-identity.spec.ts). A stopped relay still + * holds every shell it hosts — SIGSTOP suspends one process, not its children — + * so a `pty.attach` aimed at it runs out its request timeout while the shell the + * pane owns keeps running. That is the shape STA-3077 is about: a reattach that + * failed, over a session that is provably still alive. + * + * SIGCONT puts it back, so the same shell can be reattached afterwards. + */ +import { + execDockerSshRelayTargetCommand, + type DockerSshRelayTarget +} from './docker-ssh-relay-target' + +/** Guards every signal: only a detached relay may be stopped or resumed. */ +function signalDetachedRelay( + target: DockerSshRelayTarget, + relayPid: number, + signal: 'STOP' | 'CONT' +): void { + if (!Number.isInteger(relayPid) || relayPid <= 0) { + throw new Error(`Docker SSH relay process ID must be a positive integer: ${relayPid}`) + } + execDockerSshRelayTargetCommand( + target, + [ + `proc=/proc/${relayPid}`, + '[ -r "$proc/cmdline" ]', + 'argv=()', + 'mapfile -d \'\' -t argv < "$proc/cmdline"', + '[ "${argv[1]##*/}" = relay.js ]', + '[[ " ${argv[*]:2} " = *" --detached "* ]]', + `kill -${signal} ${relayPid}` + ].join(' && ') + ) +} + +export function stallDockerSshRelay(target: DockerSshRelayTarget, relayPid: number): void { + signalDetachedRelay(target, relayPid, 'STOP') + // Read the stop back off /proc rather than trusting the signal: an attach that + // is answered anyway would make every oracle downstream vacuous. + const state = execDockerSshRelayTargetCommand( + target, + `awk '/^State:/{print $2}' /proc/${relayPid}/status` + ) + if (state !== 'T') { + throw new Error(`The detached relay did not stop; /proc reports state ${state || ''}`) + } +} + +export function resumeDockerSshRelay(target: DockerSshRelayTarget, relayPid: number): void { + signalDetachedRelay(target, relayPid, 'CONT') +} + +/** True while the relay process still exists, whatever state it is in. */ +export function isDockerSshRelayProcessPresent( + target: DockerSshRelayTarget, + relayPid: number +): boolean { + return ( + execDockerSshRelayTargetCommand( + target, + `test -r /proc/${relayPid}/status && echo PRESENT || echo GONE` + ) === 'PRESENT' + ) +} diff --git a/tests/e2e/helpers/remote-pane-launch-transcript.ts b/tests/e2e/helpers/remote-pane-launch-transcript.ts new file mode 100644 index 00000000000..9bab298e604 --- /dev/null +++ b/tests/e2e/helpers/remote-pane-launch-transcript.ts @@ -0,0 +1,68 @@ +/** + * A stand-in for an agent's transcript, recorded by the host rather than by the app. + * + * The STA-3077 report's second failure is "a coding agent was resumed twice into + * one transcript". Driving a real coding agent inside the Docker OpenSSH fixture + * is not possible — no agent binary ships in the image and no provider session + * exists to resume — so the observable is moved to the thing that actually goes + * wrong: a second shell launching for a pane that already has one. Every shell + * the relay starts for a pane appends one line here, keyed by pane, so "resumed + * twice" reads as two lines for one pane key and needs no app-side reporting to + * be believed. + * + * The hook is installed before the first pane exists, and each test asserts the + * first launch was recorded before asserting no second one was — a hook that + * silently failed to fire would otherwise make every clause pass vacuously. + */ +import { + execDockerSshRelayTargetCommand, + type DockerSshRelayTarget +} from './docker-ssh-relay-target' + +const TRANSCRIPT_PATH = '/tmp/orca-pane-launch-transcript.log' + +// Prepended, not appended: Debian's stock /root/.bashrc returns early for a +// non-interactive shell, and a hook placed after that guard would only fire +// sometimes. `ORCA_PANE_KEY` is injected by the pane launch path alone, so a +// `docker exec` probe shell adds nothing. +const INSTALL_TRANSCRIPT_HOOK_COMMAND = ` +set -e +hook=/root/.orca-pane-launch-transcript.sh +cat > "$hook" <<'HOOK' +if [ -n "$ORCA_PANE_KEY" ]; then + printf '%s\\t%s\\n' "$ORCA_PANE_KEY" "$$" >> ${TRANSCRIPT_PATH} +fi +HOOK +touch /root/.bashrc ${TRANSCRIPT_PATH} +if ! grep -q orca-pane-launch-transcript /root/.bashrc; then + printf '. %s\\n' "$hook" | cat - /root/.bashrc > /root/.bashrc.next + mv /root/.bashrc.next /root/.bashrc +fi +grep -c orca-pane-launch-transcript /root/.bashrc +` + +export function installRemotePaneLaunchTranscript(target: DockerSshRelayTarget): void { + const installed = Number( + execDockerSshRelayTargetCommand(target, INSTALL_TRANSCRIPT_HOOK_COMMAND).split('\n').pop() + ) + if (installed !== 1) { + throw new Error(`The pane launch transcript hook was not installed exactly once: ${installed}`) + } +} + +/** Every shell launch the host recorded, oldest first, as `paneKey\tpid`. */ +export function readRemotePaneLaunchTranscript(target: DockerSshRelayTarget): string[] { + return execDockerSshRelayTargetCommand(target, `cat ${TRANSCRIPT_PATH} 2>/dev/null || true`) + .split('\n') + .filter((line) => line.includes('\t')) +} + +/** The pids the host launched a shell for under one pane key. */ +export function readRemotePaneLaunchPids( + target: DockerSshRelayTarget, + paneKey: string +): number[] { + return readRemotePaneLaunchTranscript(target) + .filter((line) => line.split('\t')[0] === paneKey) + .map((line) => Number(line.split('\t')[1])) +} diff --git a/tests/e2e/ssh-disconnected-pane-affordance.spec.ts b/tests/e2e/ssh-disconnected-pane-affordance.spec.ts new file mode 100644 index 00000000000..417bf56e14c --- /dev/null +++ b/tests/e2e/ssh-disconnected-pane-affordance.spec.ts @@ -0,0 +1,391 @@ +/** + * STA-3077 goalpost S3's affordance — an unreachable pane says so, offers two + * actions, and destroys nothing. + * + * Before S3, `handlePtyReattachFailure` answered any reattach failure with a + * synthetic `pty:exit { code: -1 }`, cleared provider state, deleted ownership + * and expired the lease: four claims about a process nobody had observed, on a + * shell that was usually still running. Collapsing that left the pane with no + * signal at all, so the pane now renders as disconnected with exactly two + * explicit actions — "Try again" and "Start a new terminal" — and its copy may + * never assert the shell exited, because a failed attach is not result data. + * + * ───────────────────────────────────────────────────────────────────────────── + * HELD AS `fixme`: THE STATE IS NOT INDUCIBLE FROM THE HOST TODAY. + * + * The banner needs one narrow state — the SSH target reporting itself CONNECTED + * while a single pane's `pty.attach` fails with an error that does not prove the + * session gone. `TerminalPane` suppresses this banner entirely while + * `showSshReconnectOverlay` is true, so any fault that takes the connection down + * with it produces the connection-level overlay instead and never reaches here. + * + * Two host-side faults were driven against the real Docker relay and neither + * lands in that state: + * + * 1. Stop the detached relay (SIGSTOP) and remount the pane. The relay keeps + * every shell, but the client reads the silent host as gone, reinstalls a + * relay beside it, and the SSH target drops to `connecting`. The pane's + * connect gate then DEFERS rather than attaching, so no attach fails at all; + * the pane shows "SSH connection required". Measured, not assumed: the run + * that produced this note timed out on the locator below with that overlay + * on screen. That fault is still worth having, and it is what + * ssh-reattach-does-not-resume-agent-twice.spec.ts uses to prove the + * non-destructive half of S3 — the shell stays alive, the lease stays + * claimable, and no second shell is launched. + * 2. Sever the transport. Same shape: the target leaves `connected`, the gate + * defers, and the reattach that eventually runs succeeds. + * + * What would reach it is a fault that leaves the mux healthy and fails ONE pty: + * the relay answering `sourceRecovery.restoreRequired` for a single id, which + * today only arises from a delivery-record divergence between two client + * generations. There is no seam to force that from a test without adding one in + * `src/`, which this change is not allowed to do. The oracle is kept whole and + * held rather than deleted, so it runs the day such a seam exists. + * ───────────────────────────────────────────────────────────────────────────── + * + * MUTATIONS THAT MUST REDDEN THIS FILE ONCE IT RUNS: + * - `reattach-failure-classification.ts`: make `isProvenSshSessionGoneError` + * return true for every error. The pane respawns silently, so no banner + * appears and the census gains a second shell. + * - `TerminalPaneDisconnectedBanner.tsx`: reword the `ssh-pane` copy to say the + * terminal exited, or drop either action. Clause 2 or clause 1 reddens. + * - `ssh-relay-session.ts`: restore the destructive block in + * `handlePtyReattachFailure`. Clause 3 reddens on the expired lease. + */ +import type { Locator, Page, TestInfo } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + sendToTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForPaneIdentitySnapshot, + type PaneIdentitySnapshot +} from './helpers/terminal' +import { + describeSshRemotePtyLeases, + findSshRemotePtyLeaseForLeaf, + readSshRemotePtyLeases, + resolveOrcaProfileStateFile +} from './helpers/ssh-remote-pty-lease-file' +import { + cleanupDockerSshRelayTarget, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection' +import { readDockerSshRelayProcessSnapshot } from './helpers/docker-ssh-relay-processes' +import { resumeDockerSshRelay, stallDockerSshRelay } from './helpers/docker-ssh-relay-stall' +import { + countDockerSshRelayRemoteStreamWriters, + describeDockerSshRelayRemotePtys, + readDockerSshRelayRemotePtys +} from './helpers/docker-ssh-relay-remote-ptys' +import { + installRemotePaneLaunchTranscript, + readRemotePaneLaunchPids +} from './helpers/remote-pane-launch-transcript' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' +const UNREACHABLE_PANE_INDUCTION_UNAVAILABLE = + 'No host-side fault leaves the SSH target connected while one pane attach fails; see the file header.' +// Why: the relay must outlive the fault. If it exits with the client the remote +// shells die too, and a dead shell makes "the banner overclaims" untestable. +const RELAY_GRACE_PERIOD_SECONDS = 900 +// The banner is only reachable after the pane's `pty.attach` gives up, which +// costs one mux request timeout. +const UNREACHABLE_PANE_TIMEOUT_MS = 150_000 + +/** + * The copy constraint as an oracle. A failed attach is not result data, so the + * pane may not carry a result verb (STYLEGUIDE.md:236). Enforced here rather + * than in review, because the tempting reword is exactly the old lie. + */ +const DEATH_VERB = /exit|died|dead|terminated|killed|crashed/i +/** The wire tokens the classification runs on must never reach the user. */ +const WIRE_TOKEN = /SSH_[A-Z_]+/ + +test.use({ seedTestRepo: false }) + +type UnreachableRemotePane = { + remote: Awaited> + snapshot: PaneIdentitySnapshot + paneKey: string + leafId: string + ptyId: string + pid: number + startTicks: number + relayPid: number + stateFile: string + banner: Locator +} + +/** Rebuild the pane's renderer over its live PTY — the app's own recovery seam. */ +async function remountTerminalTabForRecovery(page: Page, tabId: string): Promise { + const remounted = await page.evaluate((tabId) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('Store unavailable') + } + return state.remountTerminalTabForRecovery(tabId) + }, tabId) + expect(remounted, 'the terminal tab refused to remount, so no reattach was attempted').toBe(true) +} + +/** + * A connected remote workspace with one streaming pane whose host has stopped + * answering. Why the pane must stream: an idle pane reattaches as 'existing' and + * never enters the source re-establishment the field failure came through. + */ +async function openUnreachableRemotePane( + page: Page, + app: Parameters[0], + target: DockerSshRelayTarget +): Promise { + installRemotePaneLaunchTranscript(target) + + await waitForSessionReady(page) + const remote = await connectDockerSshRelayTarget(page, target, { + relayGracePeriodSeconds: RELAY_GRACE_PERIOD_SECONDS + }) + await expect.poll(() => waitForActiveWorktree(page), { timeout: 30_000 }).toBe(remote.worktreeId) + await ensureTerminalVisible(page, 45_000) + await waitForActiveTerminalManager(page, 60_000) + await waitForActivePanePtyId(page, 60_000) + + const snapshot = await waitForPaneIdentitySnapshot(page, 1) + const pane = snapshot.panes[0] + if (!pane?.ptyId) { + throw new Error('The remote workspace opened no pane with a PTY') + } + const paneKey = `${snapshot.tabId}:${pane.leafId}` + + const streamMarker = `SSH_AFFORDANCE_STREAM_${Date.now()}` + await sendToTerminal( + page, + pane.ptyId, + `node -e "setInterval(()=>process.stdout.write('${streamMarker}_'+Date.now()+'\\n'),100)" &\r` + ) + await expect + .poll(() => countDockerSshRelayRemoteStreamWriters(target, streamMarker), { + timeout: 60_000, + message: 'the remote pane never started streaming' + }) + .toBe(1) + + // Anti-vacuity for every "no second shell" clause below: the host records one + // launch for this pane now, so a later count of one means nothing was added. + await expect + .poll(() => readRemotePaneLaunchPids(target, paneKey).length, { + timeout: 60_000, + message: 'the host recorded no shell launch for the pane, so its transcript proves nothing' + }) + .toBe(1) + + const shell = readDockerSshRelayRemotePtys(target).find((entry) => entry.paneKey === paneKey) + if (!shell) { + throw new Error(`The relay hosts no remote shell for pane ${paneKey}`) + } + const relay = readDockerSshRelayProcessSnapshot(target) + if (!relay) { + throw new Error('No detached relay to stall') + } + + stallDockerSshRelay(target, relay.relayPid) + await remountTerminalTabForRecovery(page, snapshot.tabId) + + return { + remote, + snapshot, + paneKey, + leafId: pane.leafId, + ptyId: pane.ptyId, + pid: shell.pid, + startTicks: shell.startTicks, + relayPid: relay.relayPid, + stateFile: await resolveOrcaProfileStateFile(app), + banner: page.locator('[data-terminal-pane-disconnected-variant="ssh-pane"]') + } +} + +test.describe('an unreachable SSH pane offers two actions and destroys nothing', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH tests.') + test.skip(process.platform === 'win32', 'Docker SSH faults use POSIX SSH tooling.') + + test('the pane reports itself disconnected without claiming the shell exited', async ({ + orcaPage, + electronApp + }, testInfo: TestInfo) => { + test.fixme(true, UNREACHABLE_PANE_INDUCTION_UNAVAILABLE) + test.setTimeout(600_000) + let target: DockerSshRelayTarget | null = null + let stalledRelayPid: number | null = null + try { + target = startDockerSshRelayTarget(testInfo) + const relayTarget = target + const unreachable = await openUnreachableRemotePane(orcaPage, electronApp, relayTarget) + stalledRelayPid = unreachable.relayPid + + // 1. The pane says it is disconnected, in the SSH variant, with both actions. + await expect( + unreachable.banner, + 'the unreachable pane showed no disconnected affordance' + ).toBeVisible({ timeout: UNREACHABLE_PANE_TIMEOUT_MS }) + await expect( + unreachable.banner, + 'the disconnected pane is still advertising automatic retries' + ).toHaveAttribute('data-terminal-remote-runtime-reconnect-banner', 'disconnected') + await expect(unreachable.banner.getByRole('button', { name: 'Try again' })).toBeVisible() + await expect( + unreachable.banner.getByRole('button', { name: 'Start a new terminal' }) + ).toBeVisible() + + // 2. Copy constraint. Nothing here observed an exit, so nothing may report one. + const copy = (await unreachable.banner.textContent()) ?? '' + expect(copy.trim(), 'the disconnected banner rendered no copy at all').not.toBe('') + expect(copy, 'the disconnected banner claims the shell is gone').not.toMatch(DEATH_VERB) + expect(copy, 'the disconnected banner leaks a wire token').not.toMatch(WIRE_TOKEN) + testInfo.annotations.push({ type: 'ssh-disconnected-pane-copy', description: copy }) + + // 3. The claim under the copy: the shell is alive and the lease is not retired. + const shellsWhileUnreachable = readDockerSshRelayRemotePtys(relayTarget) + expect( + shellsWhileUnreachable.filter( + (shell) => shell.pid === unreachable.pid && shell.startTicks === unreachable.startTicks + ), + 'the failed reattach killed the shell the pane owns' + ).toHaveLength(1) + const lease = findSshRemotePtyLeaseForLeaf( + unreachable.stateFile, + unreachable.remote.targetId, + unreachable.leafId + ) + expect(lease, 'the failed reattach dropped the pane lease').toBeDefined() + expect(lease?.state, 'the failed reattach expired a lease over a running shell').not.toBe( + 'expired' + ) + expect(lease?.state, 'the failed reattach retired a lease over a running shell').not.toBe( + 'terminated' + ) + expect( + readRemotePaneLaunchPids(relayTarget, unreachable.paneKey), + 'the failed reattach launched a second shell for the pane' + ).toHaveLength(1) + testInfo.annotations.push({ + type: 'ssh-disconnected-pane-unreachable', + description: `${describeDockerSshRelayRemotePtys(shellsWhileUnreachable)} leases=${describeSshRemotePtyLeases( + readSshRemotePtyLeases(unreachable.stateFile, unreachable.remote.targetId) + )}` + }) + + // 4. 'Try again' reattaches the same shell once the host can answer again. + resumeDockerSshRelay(relayTarget, unreachable.relayPid) + stalledRelayPid = null + await unreachable.banner.getByRole('button', { name: 'Try again' }).click() + await expect(unreachable.banner, 'the banner outlived a successful retry').toBeHidden({ + timeout: UNREACHABLE_PANE_TIMEOUT_MS + }) + await waitForActiveTerminalManager(orcaPage, 60_000) + const recovered = await waitForPaneIdentitySnapshot(orcaPage, 1) + expect(recovered.panes[0]?.ptyId, 'the retry bound the pane to a different session').toBe( + unreachable.ptyId + ) + expect( + readDockerSshRelayRemotePtys(relayTarget).map( + (shell) => `${shell.paneKey ?? '-'}=${shell.pid}@${shell.startTicks}` + ), + 'the retry added or replaced a remote shell' + ).toEqual([`${unreachable.paneKey}=${unreachable.pid}@${unreachable.startTicks}`]) + expect( + readRemotePaneLaunchPids(relayTarget, unreachable.paneKey), + 'the retry launched a second shell for the pane' + ).toHaveLength(1) + } finally { + if (target && stalledRelayPid !== null) { + resumeDockerSshRelay(target, stalledRelayPid) + } + cleanupDockerSshRelayTarget(target) + } + }) + + // The other action, and the reason it is not styled as a loss: it adds a shell + // rather than replacing one, and the pane the user had is still the pane the + // user has. + test('starting a new terminal adds exactly one shell and leaves the panes alone', async ({ + orcaPage, + electronApp + }, testInfo: TestInfo) => { + test.fixme(true, UNREACHABLE_PANE_INDUCTION_UNAVAILABLE) + test.setTimeout(600_000) + let target: DockerSshRelayTarget | null = null + let stalledRelayPid: number | null = null + try { + target = startDockerSshRelayTarget(testInfo) + const relayTarget = target + const unreachable = await openUnreachableRemotePane(orcaPage, electronApp, relayTarget) + stalledRelayPid = unreachable.relayPid + + await expect( + unreachable.banner, + 'the unreachable pane showed no disconnected affordance' + ).toBeVisible({ timeout: UNREACHABLE_PANE_TIMEOUT_MS }) + + resumeDockerSshRelay(relayTarget, unreachable.relayPid) + stalledRelayPid = null + await unreachable.banner.getByRole('button', { name: 'Start a new terminal' }).click() + + // Exactly one more shell: the one the user asked for. The old shell is left + // running, which is what makes "starting a new one leaves it alone" true. + await expect + .poll(() => readRemotePaneLaunchPids(relayTarget, unreachable.paneKey).length, { + timeout: UNREACHABLE_PANE_TIMEOUT_MS, + message: 'starting a new terminal did not settle at exactly one additional launch' + }) + .toBe(2) + expect( + readDockerSshRelayRemotePtys(relayTarget).filter( + (shell) => shell.pid === unreachable.pid && shell.startTicks === unreachable.startTicks + ), + 'starting a new terminal killed the shell it promised to leave alone' + ).toHaveLength(1) + expect( + readDockerSshRelayRemotePtys(relayTarget), + 'starting a new terminal changed the host by more than one shell' + ).toHaveLength(2) + + // The pane count is a user-visible promise: a new shell is not a new pane. + const paneCount = await orcaPage.evaluate((worktreeId) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('Store unavailable') + } + type LayoutNode = + | { type: 'leaf'; leafId: string } + | { type: 'split'; first: LayoutNode; second: LayoutNode } + | null + const collectLeafIds = (node: LayoutNode): string[] => + !node + ? [] + : node.type === 'leaf' + ? [node.leafId] + : [...collectLeafIds(node.first), ...collectLeafIds(node.second)] + return (state.tabsByWorktree[worktreeId] ?? []).reduce((total, tab) => { + const leaves = collectLeafIds( + (state.terminalLayoutsByTabId[tab.id]?.root ?? null) as LayoutNode + ) + return total + Math.max(leaves.length, 1) + }, 0) + }, unreachable.remote.worktreeId) + expect(paneCount, 'starting a new terminal surfaced a pane the user never opened').toBe(1) + testInfo.annotations.push({ + type: 'ssh-disconnected-pane-new-terminal', + description: describeDockerSshRelayRemotePtys(readDockerSshRelayRemotePtys(relayTarget)) + }) + } finally { + if (target && stalledRelayPid !== null) { + resumeDockerSshRelay(target, stalledRelayPid) + } + cleanupDockerSshRelayTarget(target) + } + }) +}) diff --git a/tests/e2e/ssh-reattach-does-not-resume-agent-twice.spec.ts b/tests/e2e/ssh-reattach-does-not-resume-agent-twice.spec.ts new file mode 100644 index 00000000000..35f97894f58 --- /dev/null +++ b/tests/e2e/ssh-reattach-does-not-resume-agent-twice.spec.ts @@ -0,0 +1,244 @@ +/** + * STA-3077 goalpost S3 — a reattach failure may not become a respawn — and S8, + * which removed the last path that granted one without being asked. + * + * The reported failure: a coding agent was resumed twice into one transcript. + * The mechanism is a converging decision. A reattach that failed for any reason + * was read as proof the session had gone, so `handlePtyReattachFailure` sent the + * pane a synthetic `pty:exit { code: -1 }`, the pane read that as a death, and + * the cold-restore resume ran with the same provider session id over a shell + * that was still running. Two agent processes then appended to one transcript. + * Respawn now requires proof; everything else stays unresolved. + * + * WHAT IS PROXIED, AND WHY. A real coding agent cannot be driven here: the + * Docker OpenSSH fixture ships no agent binary and there is no provider session + * to resume, so "one transcript, two resumes" has no literal form in this + * harness. The observable moves one level down, to the thing that must happen + * before an agent can be resumed twice — a SECOND shell launching for a pane + * that already has one. The host records every launch itself + * (helpers/remote-pane-launch-transcript.ts), so the count is taken on the + * container and owes the app nothing. + * + * THE FAULT. The detached relay is stopped, not killed. It keeps every shell it + * hosts (SIGSTOP suspends one process, not its children) and answers nothing. + * The client treats the silent host as gone and reinstalls a relay beside it, so + * the pane's session is one the new relay has never heard of, over a shell that + * is provably still running on the container — which is exactly the state the + * fabricated exit used to lie about. + * + * MUTATIONS THAT MUST REDDEN THIS FILE: + * - `src/main/ssh/ssh-relay-session.ts`: restore the destructive block in + * `handlePtyReattachFailure`. The fabricated exit reaches the pane, which + * cold-restores over a live shell and the host records a second launch. + * - `src/renderer/.../reattach-failure-classification.ts`: make + * `isProvenSshSessionGoneError` return true for every error, and the pane's + * own reattach arm respawns instead of holding. + * - `src/main/runtime/orca-runtime.ts`: reinstate the recovery grant S8 deleted + * from `recoverTerminalPane`, with its id comparison normalized so it can + * actually fire; a disconnected pane then starts a replacement on its own. + */ +import type { Page, TestInfo } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + sendToTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForPaneIdentitySnapshot +} from './helpers/terminal' +import { + findSshRemotePtyLeaseForLeaf, + describeSshRemotePtyLeases, + readSshRemotePtyLeases, + resolveOrcaProfileStateFile +} from './helpers/ssh-remote-pty-lease-file' +import { + cleanupDockerSshRelayTarget, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection' +import { readDockerSshRelayProcessSnapshot } from './helpers/docker-ssh-relay-processes' +import { resumeDockerSshRelay, stallDockerSshRelay } from './helpers/docker-ssh-relay-stall' +import { + countDockerSshRelayRemoteStreamWriters, + describeDockerSshRelayRemotePtys, + readDockerSshRelayRemotePtys +} from './helpers/docker-ssh-relay-remote-ptys' +import { + installRemotePaneLaunchTranscript, + readRemotePaneLaunchPids +} from './helpers/remote-pane-launch-transcript' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' +// Why: the relay must outlive the fault. If it exits with the client the remote +// shells die too, and a respawn over a corpse is not the defect this bounds. +const RELAY_GRACE_PERIOD_SECONDS = 900 +// The pane has to give up on a host that answers nothing, which costs a mux +// request timeout before anything is classified. +const UNREACHABLE_PANE_TIMEOUT_MS = 180_000 +// A respawn can land late, after the failure is classified. One sample would +// miss it, so every clause is held over a window instead. +const HOLD_WINDOW_MS = 25_000 + +test.use({ seedTestRepo: false }) + +/** Rebuild the pane's renderer over its live PTY — the app's own recovery seam. */ +async function remountTerminalTabForRecovery(page: Page, tabId: string): Promise { + const remounted = await page.evaluate((tabId) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('Store unavailable') + } + return state.remountTerminalTabForRecovery(tabId) + }, tabId) + expect(remounted, 'the terminal tab refused to remount, so no reattach was attempted').toBe(true) +} + +test.describe('a failed SSH reattach never starts a second shell for the pane', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH tests.') + test.skip(process.platform === 'win32', 'Docker SSH faults use POSIX SSH tooling.') + + test('a host that stops answering leaves the pane on the shell it already had', async ({ + orcaPage, + electronApp + }, testInfo: TestInfo) => { + test.setTimeout(600_000) + let target: DockerSshRelayTarget | null = null + let stalledRelayPid: number | null = null + try { + target = startDockerSshRelayTarget(testInfo) + const relayTarget = target + installRemotePaneLaunchTranscript(relayTarget) + + await waitForSessionReady(orcaPage) + const remote = await connectDockerSshRelayTarget(orcaPage, relayTarget, { + relayGracePeriodSeconds: RELAY_GRACE_PERIOD_SECONDS + }) + await expect + .poll(() => waitForActiveWorktree(orcaPage), { timeout: 30_000 }) + .toBe(remote.worktreeId) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + await waitForActivePanePtyId(orcaPage, 60_000) + + const snapshot = await waitForPaneIdentitySnapshot(orcaPage, 1) + const pane = snapshot.panes[0] + if (!pane?.ptyId) { + throw new Error('The remote workspace opened no pane with a PTY') + } + const paneKey = `${snapshot.tabId}:${pane.leafId}` + const stateFile = await resolveOrcaProfileStateFile(electronApp) + + // Why the pane must stream: an idle pane reattaches as 'existing' and never + // enters the source re-establishment the field failure came through. + const streamMarker = `SSH_RESUME_ONCE_STREAM_${Date.now()}` + await sendToTerminal( + orcaPage, + pane.ptyId, + `node -e "setInterval(()=>process.stdout.write('${streamMarker}_'+Date.now()+'\\n'),100)" &\r` + ) + await expect + .poll(() => countDockerSshRelayRemoteStreamWriters(relayTarget, streamMarker), { + timeout: 60_000, + message: 'the remote pane never started streaming' + }) + .toBe(1) + + // Anti-vacuity. Every clause below counts launches; had the host recorded + // none, "still exactly one" would prove nothing at all. + await expect + .poll(() => readRemotePaneLaunchPids(relayTarget, paneKey).length, { + timeout: 60_000, + message: 'the host recorded no shell launch for the pane, so its transcript is inert' + }) + .toBe(1) + const launchPid = readRemotePaneLaunchPids(relayTarget, paneKey)[0] + const shell = readDockerSshRelayRemotePtys(relayTarget).find( + (entry) => entry.paneKey === paneKey + ) + if (!shell) { + throw new Error(`The relay hosts no remote shell for pane ${paneKey}`) + } + // Cross-checks the transcript against the process census: the pid the host + // logged at launch is the pid the relay hosts for this pane. + expect( + shell.pid, + 'the launch the host recorded is not the shell the relay hosts for this pane' + ).toBe(launchPid) + const shellIdentity = `${shell.pid}@${shell.startTicks}` + testInfo.annotations.push({ + type: 'ssh-resume-once-baseline', + description: `${describeDockerSshRelayRemotePtys(readDockerSshRelayRemotePtys(relayTarget))} launches=1` + }) + + const relay = readDockerSshRelayProcessSnapshot(relayTarget) + if (!relay) { + throw new Error('No detached relay to stall') + } + stallDockerSshRelay(relayTarget, relay.relayPid) + stalledRelayPid = relay.relayPid + await remountTerminalTabForRecovery(orcaPage, snapshot.tabId) + + // Vacuity guard: the pane has to actually lose its host before "it kept its + // shell" means anything. This overlay is the user-visible proof it did — + // without it, a pane that quietly reattached would pass every clause below. + await expect( + orcaPage.locator('[data-terminal-ssh-reconnect-banner]').first(), + 'the pane never lost its host, so the clauses below are vacuous' + ).toBeVisible({ timeout: UNREACHABLE_PANE_TIMEOUT_MS }) + + /** Everything a duplicate resume would move, read from the host and from disk. */ + const expectNoSecondShell = (label: string): void => { + expect( + readRemotePaneLaunchPids(relayTarget, paneKey), + `${label} launched a second shell for pane ${paneKey}` + ).toEqual([launchPid]) + expect( + readDockerSshRelayRemotePtys(relayTarget) + .filter((entry) => entry.paneKey === paneKey) + .map((entry) => `${entry.pid}@${entry.startTicks}`), + `${label} killed or replaced the shell the pane owns` + ).toEqual([shellIdentity]) + // Unknown is not dead: the lease must stay claimable so a later reattach + // can still find this shell. + const lease = findSshRemotePtyLeaseForLeaf(stateFile, remote.targetId, pane.leafId) + expect(lease, `${label} dropped the pane lease`).toBeDefined() + expect(lease?.state, `${label} expired a lease over a running shell`).not.toBe('expired') + expect(lease?.state, `${label} retired a lease over a running shell`).not.toBe('terminated') + } + + // Sample continuously rather than poll-until-equal: a respawn that lands + // and is later tidied up would satisfy a poll needing one matching read. + const deadline = Date.now() + HOLD_WINDOW_MS + let samples = 0 + while (Date.now() < deadline) { + expectNoSecondShell('the unreachable host') + samples += 1 + await orcaPage.waitForTimeout(500) + } + expect(samples).toBeGreaterThan(10) + + // The host comes back and nothing is granted automatically: a disconnected + // pane refuses to respawn on its own (S8), so the only route to a second + // shell is the user asking for one. + resumeDockerSshRelay(relayTarget, relay.relayPid) + stalledRelayPid = null + await orcaPage.waitForTimeout(HOLD_WINDOW_MS) + expectNoSecondShell('the host returning') + testInfo.annotations.push({ + type: 'ssh-resume-once-settled', + description: `${describeDockerSshRelayRemotePtys( + readDockerSshRelayRemotePtys(relayTarget) + )} launches=${readRemotePaneLaunchPids(relayTarget, paneKey).length} leases=${describeSshRemotePtyLeases( + readSshRemotePtyLeases(stateFile, remote.targetId) + )}` + }) + } finally { + if (target && stalledRelayPid !== null) { + resumeDockerSshRelay(target, stalledRelayPid) + } + cleanupDockerSshRelayTarget(target) + } + }) +}) diff --git a/tests/e2e/ssh-reconnect-pane-cardinality-across-partitions.spec.ts b/tests/e2e/ssh-reconnect-pane-cardinality-across-partitions.spec.ts new file mode 100644 index 00000000000..2b8138248d5 --- /dev/null +++ b/tests/e2e/ssh-reconnect-pane-cardinality-across-partitions.spec.ts @@ -0,0 +1,349 @@ +/** + * STA-3077 goalpost S4 — one durable partition per (target, pane). + * + * The reported failure: reconnecting an SSH host multiplied the user's + * terminals, 2 -> 19 -> 20 across three reconnects. The mechanism was a pane + * binding with two homes. Main's spawn wrote `ssh:`, the relay's + * reattach write passed no hostId and landed in `local`, and the renderer has + * always published SSH pane membership to `local`. Supersession read the + * partition no live writer maintained, saw the arriving lease disagree with a + * stale pty id, and bailed — so it silently no-opped and every reconnect left + * another live lease behind, with a shell and a pane to match. + * + * The mutation that must redden this file: in `src/main/ipc/pty.ts`, restore + * the ssh-first-then-local hedge in `durablyBoundPtyIdForPane` and give the + * spawn upserts their `ssh:` hostId back. The divergent copy then + * reappears, supersession stops firing, and `distinctBoundPtyIdsByLeaf` / + * `liveLeasesByLeaf` grow past one per pane. + * + * Why this spec exists beside ssh-reconnect-pane-cardinality.spec.ts: that file + * faults the transport with a kill, which reconnects with every lease still + * marked attached. The user reported *reconnecting the host*, which is a clean + * disconnect followed by a connect. Only the detach path moves a lease + * attached -> detached and runs supersession on the way back, so this is the + * shape the defect actually lived on, and it is the shape driven here — three + * times, because the report counted three reconnects. + * + * Every clause is an observable: panes the user can see, leases on disk, + * bindings in the durable session, and shells counted on the container itself. + */ +import type { Page, TestInfo } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + sendToTerminal, + splitActiveTerminalPane, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForPaneIdentitySnapshot, + type PaneIdentitySnapshot +} from './helpers/terminal' +import { readDurablePaneBindings, sshExecutionHostId } from './helpers/remote-pane-durable-session' +import { + describeSshRemotePtyLeases, + readSshRemotePtyLeases, + resolveOrcaProfileStateFile +} from './helpers/ssh-remote-pty-lease-file' +import { + cleanupDockerSshRelayTarget, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { + connectDockerSshRelayTarget, + reconnectDockerSshRelayTarget +} from './helpers/docker-ssh-relay-connection' +import { + countDockerSshRelayRemoteStreamWriters, + describeDockerSshRelayRemotePtys, + readDockerSshRelayRemotePtys +} from './helpers/docker-ssh-relay-remote-ptys' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' +const PANE_COUNT = 2 +// The reported repro shape: 2 -> 19 -> 20 was counted over three reconnects. +const RECONNECT_CYCLES = 3 +// Why: the relay must outlive every reconnect. If it exits with the client the +// remote shells die too and reconnect degrades to a cold spawn, which is not +// the path this spec bounds. +const RELAY_GRACE_PERIOD_SECONDS = 900 +// Why: a graft lands after reattach reports ready, so every dimension is +// re-read once the dust settles rather than the instant a wait passes. +const SETTLE_MS = 6_000 + +test.use({ seedTestRepo: false }) + +/** Leases that could still claim a shell. Terminated and expired ones cannot. */ +function liveLeasesByLeaf(stateFile: string, targetId: string): Record { + const byLeaf: Record = {} + for (const lease of readSshRemotePtyLeases(stateFile, targetId)) { + if (lease.state === 'terminated' || lease.state === 'expired') { + continue + } + const leafId = lease.leafId ?? '' + byLeaf[leafId] = [...(byLeaf[leafId] ?? []), `${lease.ptyId}=${lease.state}`].sort() + } + return byLeaf +} + +/** + * `leafId -> the distinct pty ids any durable partition binds it to`. + * + * Two partitions naming the same leaf is tolerable while they agree; naming it + * twice with different shells is the divergence supersession failed to resolve, + * and it is what a user reads as a pane that came back doubled. + */ +async function distinctBoundPtyIdsByLeaf( + page: Page, + hostId: string, + worktreeId: string +): Promise> { + const byLeaf: Record> = {} + for (const binding of await readDurablePaneBindings(page, hostId, worktreeId)) { + // ` /=` + const [, pane] = binding.split(' ') + const [paneKey, ptyId] = (pane ?? '').split('=') + const leafId = (paneKey ?? '').split('/')[1] + if (!leafId || !ptyId) { + throw new Error(`Unparsable durable pane binding: ${binding}`) + } + byLeaf[leafId] = (byLeaf[leafId] ?? new Set()).add(ptyId) + } + return Object.fromEntries( + Object.entries(byLeaf).map(([leafId, ptyIds]) => [leafId, [...ptyIds].sort()]) + ) +} + +/** `tabId/leafId` for every pane the user can see in the workspace, plus its tabs. */ +async function readRemotePaneCensus( + page: Page, + worktreeId: string +): Promise<{ tabIds: string[]; paneIds: string[] }> { + return page.evaluate((worktreeId) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('Store unavailable') + } + type LayoutNode = + | { type: 'leaf'; leafId: string } + | { type: 'split'; first: LayoutNode; second: LayoutNode } + | null + const collectLeafIds = (node: LayoutNode): string[] => + !node + ? [] + : node.type === 'leaf' + ? [node.leafId] + : [...collectLeafIds(node.first), ...collectLeafIds(node.second)] + const tabs = state.tabsByWorktree[worktreeId] ?? [] + const paneIds = tabs.flatMap((tab) => { + const leafIds = collectLeafIds( + (state.terminalLayoutsByTabId[tab.id]?.root ?? null) as LayoutNode + ) + // `root: null` is the implicit single-pane layout a fresh tab carries + // until it is first split, so it still counts as one visible pane. + return leafIds.length > 0 + ? leafIds.map((leafId) => `${tab.id}/${leafId}`) + : [`${tab.id}/`] + }) + return { tabIds: tabs.map((tab) => tab.id).sort(), paneIds: paneIds.sort() } + }, worktreeId) +} + +async function waitForSshConnected(page: Page, targetId: string, timeoutMs: number): Promise { + await expect + .poll( + () => + page.evaluate( + async (targetId) => (await window.api.ssh.getState({ targetId }))?.status ?? null, + targetId + ), + { timeout: timeoutMs, message: 'SSH target never reported itself connected' } + ) + .toBe('connected') +} + +/** + * A connected remote workspace with PANE_COUNT panes, each carrying a live + * output source. Why every pane must stream: an idle pane reattaches as + * 'existing' and never enters the source re-establishment the field failure + * came through. The writer is backgrounded so the shell keeps its prompt. + */ +async function openStreamingRemotePanes( + page: Page, + target: DockerSshRelayTarget +): Promise<{ + remote: Awaited> + snapshot: PaneIdentitySnapshot +}> { + await waitForSessionReady(page) + const remote = await connectDockerSshRelayTarget(page, target, { + relayGracePeriodSeconds: RELAY_GRACE_PERIOD_SECONDS + }) + await expect.poll(() => waitForActiveWorktree(page), { timeout: 30_000 }).toBe(remote.worktreeId) + await ensureTerminalVisible(page, 45_000) + await waitForActiveTerminalManager(page, 60_000) + await waitForActivePanePtyId(page, 60_000) + + await splitActiveTerminalPane(page, 'vertical') + const snapshot = await waitForPaneIdentitySnapshot(page, PANE_COUNT) + + const streamMarker = `SSH_PARTITION_STREAM_${Date.now()}` + for (const pane of snapshot.panes) { + if (!pane.ptyId) { + throw new Error(`Pane ${pane.leafId} has no PTY to stream from`) + } + await sendToTerminal( + page, + pane.ptyId, + `node -e "setInterval(()=>process.stdout.write('${streamMarker}_'+Date.now()+'\\n'),100)" &\r` + ) + } + await expect + .poll(() => countDockerSshRelayRemoteStreamWriters(target, streamMarker), { + timeout: 60_000, + message: 'remote panes did not start streaming before the first reconnect' + }) + .toBe(PANE_COUNT) + await expect + .poll(() => readDockerSshRelayRemotePtys(target).length, { + timeout: 60_000, + message: 'remote shells did not settle at one per pane before the first reconnect' + }) + .toBe(PANE_COUNT) + return { remote, snapshot } +} + +test.describe('SSH reconnect cardinality with one durable partition per pane', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH tests.') + test.skip(process.platform === 'win32', 'Docker SSH reconnect uses POSIX SSH tooling.') + + test('three reconnects add no pane, no live lease and no remote shell', async ({ + orcaPage, + electronApp + }, testInfo: TestInfo) => { + test.setTimeout(600_000) + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + const relayTarget = target + const { remote, snapshot } = await openStreamingRemotePanes(orcaPage, relayTarget) + const hostId = sshExecutionHostId(remote.targetId) + const stateFile = await resolveOrcaProfileStateFile(electronApp) + const leafIds = snapshot.panes.map((pane) => pane.leafId) + const paneKeys = leafIds.map((leafId) => `${snapshot.tabId}:${leafId}`) + const shellIdentity = (shell: { paneKey: string | null; pid: number; startTicks: number }) => + `${shell.paneKey ?? '-'}=${shell.pid}@${shell.startTicks}` + + const baselinePanes = await readRemotePaneCensus(orcaPage, remote.worktreeId) + expect(baselinePanes.paneIds).toHaveLength(PANE_COUNT) + const baselineShells = readDockerSshRelayRemotePtys(relayTarget) + // Ties every later clause to a real starting point: the whole spec is + // vacuous if the panes never owned a shell in the first place. + for (const paneKey of paneKeys) { + expect( + baselineShells.filter((shell) => shell.paneKey === paneKey), + `pane ${paneKey} did not start out owning exactly one remote shell` + ).toHaveLength(1) + } + const baselineBindings = await distinctBoundPtyIdsByLeaf(orcaPage, hostId, remote.worktreeId) + for (const leafId of leafIds) { + expect( + baselineBindings[leafId] ?? [], + `pane ${leafId} must start out durably bound to exactly one shell` + ).toHaveLength(1) + } + testInfo.annotations.push({ + type: 'ssh-partition-cardinality-baseline', + description: `${describeDockerSshRelayRemotePtys(baselineShells)} leases=${describeSshRemotePtyLeases( + readSshRemotePtyLeases(stateFile, remote.targetId) + )}` + }) + + /** Every dimension a duplicated pane binding moves, re-read from its owner. */ + const assertCardinalityHolds = async (label: string): Promise => { + const panes = await readRemotePaneCensus(orcaPage, remote.worktreeId) + expect(panes.paneIds, `${label} changed the panes the user can see`).toEqual( + baselinePanes.paneIds + ) + expect(panes.tabIds, `${label} changed the workspace tabs`).toEqual(baselinePanes.tabIds) + + const leases = liveLeasesByLeaf(stateFile, remote.targetId) + for (const leafId of leafIds) { + expect( + leases[leafId] ?? [], + `${label} left more than one live lease naming pane ${snapshot.tabId}/${leafId}` + ).toHaveLength(1) + } + expect( + Object.keys(leases).sort(), + `${label} left a live lease for a pane that does not exist` + ).toEqual([...leafIds].sort()) + + const shells = readDockerSshRelayRemotePtys(relayTarget) + for (const paneKey of paneKeys) { + expect( + shells.filter((shell) => shell.paneKey === paneKey), + `${label} left the host running more than one shell for pane ${paneKey}` + ).toHaveLength(1) + } + expect(shells, `${label} changed the number of shells the host runs`).toHaveLength( + PANE_COUNT + ) + + // The pane may legitimately be bound to a successor shell, but never to + // two at once: that is the divergence a user reads as a doubled terminal. + const bindings = await distinctBoundPtyIdsByLeaf(orcaPage, hostId, remote.worktreeId) + for (const leafId of leafIds) { + expect( + bindings[leafId] ?? [], + `${label} left pane ${leafId} durably bound to more than one shell` + ).toHaveLength(1) + } + expect( + Object.keys(bindings).sort(), + `${label} durably bound a pane the user never opened` + ).toEqual([...leafIds].sort()) + } + + for (let cycle = 1; cycle <= RECONNECT_CYCLES; cycle += 1) { + await reconnectDockerSshRelayTarget(orcaPage, remote.targetId) + await waitForSshConnected(orcaPage, remote.targetId, 180_000) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + await waitForActivePanePtyId(orcaPage, 60_000) + + // Poll first: a duplicate can land after reattach reports ready, and a + // single read taken the instant the wait passes would miss it. + await expect + .poll(() => readDockerSshRelayRemotePtys(relayTarget).length, { + timeout: 120_000, + message: `reconnect ${cycle} changed the number of shells the host runs` + }) + .toBe(PANE_COUNT) + await assertCardinalityHolds(`reconnect ${cycle}`) + + // Poll returns on its first passing probe, so sit out the settle window + // and re-read every dimension a late graft could still move. Attributing + // it to the reconnect that caused it beats letting it leak into the next + // cycle, where the count is already suspect. + await orcaPage.waitForTimeout(SETTLE_MS) + await assertCardinalityHolds(`reconnect ${cycle} after settling`) + testInfo.annotations.push({ + type: `ssh-partition-cardinality-reconnect-${cycle}`, + description: `${describeDockerSshRelayRemotePtys( + readDockerSshRelayRemotePtys(relayTarget) + )} leases=${describeSshRemotePtyLeases(readSshRemotePtyLeases(stateFile, remote.targetId))}` + }) + } + + // Closing the loop on the report itself: after the third reconnect the + // host runs exactly the shells it ran before the first one. + expect( + readDockerSshRelayRemotePtys(relayTarget).map(shellIdentity).sort(), + `${RECONNECT_CYCLES} reconnects replaced or multiplied the remote shells` + ).toEqual(baselineShells.map(shellIdentity).sort()) + } finally { + cleanupDockerSshRelayTarget(target) + } + }) +})