From 0dbe9d050488e64d01ec4dfdeb8184fdbb72ff75 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:35:47 -0700 Subject: [PATCH] test(ssh): dockerized relay fault injection with verdict assertions (#18017) * test(ssh): add a dockerized SSH fault-injection lane with four fault shapes The existing SSH reconnect specs all reconnect by calling ssh.disconnect() then ssh.connect() - a clean cycle the client knows is coming. Nothing covered the faults the reconnect machinery exists for. Four shapes, each documented with why it is not the others: killing sshd's per-connection forks (transport dies, relay survives), `docker pause` (silence with TCP still established), SIGKILLing every relay.js (the only fault where `exited` is the correct verdict), and a 48MB flood with nobody attached. The relay-kill case is the one that makes the rest meaningful: every other case asserts the session survived, which only means something if a genuinely dead session is distinguishable. It is the only case where replacing the pane is correct, so it pins the boundary in docs/reference/ssh-execution-boundary.md rather than just testing reconnection. The `docker pause` case pins the other side of that boundary: after 30s of silence from a healthy host the pane keeps its PTY and its scrollback, because loss of contact is never evidence of death. No network-blackhole fault: reconnecting the fixture does not restore its published port mapping, so that fault is not reversible on this container and would strand the worker it ran on. * test(ssh): fixme the flood case pending #18018 It fails in CI on its first real run: the pane keeps its PTY and repaints, but a command run after the flood produces no output within the poll budget. Same shape as #18018 and not caused by this spec. The three verdict assertions around it stay enforced. --- config/scripts/pr-e2e-gate-contract.test.mjs | 1 + config/scripts/pr-e2e-source-routing.mjs | 1 + config/scripts/run-ssh-docker-e2e.mjs | 1 + tests/e2e/helpers/docker-ssh-relay-faults.ts | 133 +++++++ ...ssh-docker-transport-drop-recovery.spec.ts | 346 ++++++++++++++++++ 5 files changed, 482 insertions(+) create mode 100644 tests/e2e/helpers/docker-ssh-relay-faults.ts create mode 100644 tests/e2e/ssh-docker-transport-drop-recovery.spec.ts diff --git a/config/scripts/pr-e2e-gate-contract.test.mjs b/config/scripts/pr-e2e-gate-contract.test.mjs index 7cf2b4e11b4..232780c6941 100644 --- a/config/scripts/pr-e2e-gate-contract.test.mjs +++ b/config/scripts/pr-e2e-gate-contract.test.mjs @@ -266,6 +266,7 @@ describe('PR E2E gate contract', () => { 'tests/e2e/pty-input-write-queue-ssh.spec.ts', 'tests/e2e/ssh-cold-activation-restore.spec.ts', 'tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts', + 'tests/e2e/ssh-docker-transport-drop-recovery.spec.ts', 'tests/e2e/ssh-port-forward-lifecycle.spec.ts', 'tests/e2e/ssh-reconnect-tab-destruction.spec.ts', 'tests/e2e/ssh-startup-exec-readiness.spec.ts', diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs index 1aed38db92e..d81f4c040fe 100644 --- a/config/scripts/pr-e2e-source-routing.mjs +++ b/config/scripts/pr-e2e-source-routing.mjs @@ -27,6 +27,7 @@ export const PR_E2E_SOURCE_ROUTES = [ 'tests/e2e/pty-input-write-queue-ssh.spec.ts', 'tests/e2e/ssh-cold-activation-restore.spec.ts', 'tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts', + 'tests/e2e/ssh-docker-transport-drop-recovery.spec.ts', 'tests/e2e/ssh-port-forward-lifecycle.spec.ts', 'tests/e2e/ssh-reconnect-tab-destruction.spec.ts', 'tests/e2e/ssh-startup-exec-readiness.spec.ts', diff --git a/config/scripts/run-ssh-docker-e2e.mjs b/config/scripts/run-ssh-docker-e2e.mjs index a723a9a6ad0..195811f7312 100644 --- a/config/scripts/run-ssh-docker-e2e.mjs +++ b/config/scripts/run-ssh-docker-e2e.mjs @@ -72,6 +72,7 @@ const result = spawnSync( 'tests/e2e/ssh-cold-activation-restore.spec.ts', 'tests/e2e/ssh-cold-hydration-gap-tab-seeding.spec.ts', 'tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts', + 'tests/e2e/ssh-docker-transport-drop-recovery.spec.ts', 'tests/e2e/ssh-external-image-preview.spec.ts', 'tests/e2e/ssh-lost-kill-tab-resurrection.spec.ts', 'tests/e2e/ssh-pi-compatible-agent-title.spec.ts', diff --git a/tests/e2e/helpers/docker-ssh-relay-faults.ts b/tests/e2e/helpers/docker-ssh-relay-faults.ts new file mode 100644 index 00000000000..f7c8f77fb2f --- /dev/null +++ b/tests/e2e/helpers/docker-ssh-relay-faults.ts @@ -0,0 +1,133 @@ +import { execFileSync, spawnSync } from 'node:child_process' +import { + execDockerSshRelayTargetControlCommand, + type DockerSshRelayTarget +} from './docker-ssh-relay-target' + +function run(args: string[], opts: { timeoutMs?: number } = {}): string { + return execFileSync('docker', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: opts.timeoutMs ?? 30_000 + }).trim() +} + +function tryRun(args: string[], opts: { timeoutMs?: number } = {}): boolean { + return ( + spawnSync('docker', args, { + stdio: 'ignore', + timeout: opts.timeoutMs ?? 10_000 + }).status === 0 + ) +} + +/** + * Kill the per-connection sshd forks, leaving the listening daemon and every relay + * process alive. + * + * Why: this is the fault the reconnect path is actually built for — the transport + * dies while the remote session is still running, so a correct client re-attaches + * rather than redeploying. Killing the container or the daemon tests a different + * thing (see killDockerSshRelayDaemon / blackholeDockerSshRelayNetwork). + */ +export function dropDockerSshRelayTransport(target: DockerSshRelayTarget): number { + // Why: the listener is the oldest sshd (PID 1 under the fixture entrypoint); every + // other sshd/sshd-session is a live connection. OpenSSH >= 9.8 renames the child, + // so both names are matched to keep this working across fixture image bumps. + const output = execDockerSshRelayTargetControlCommand( + target, + ` +daemon="$(pgrep -x sshd | sort -n | head -1)" +[ -n "$daemon" ] || { echo 0; exit 0; } +killed=0 +for pid in $(pgrep -x sshd; pgrep -x sshd-session); do + [ "$pid" = "$daemon" ] && continue + kill -9 "$pid" 2>/dev/null && killed=$((killed+1)) +done +echo "$killed" +` + ) + const dropped = Number(output.trim().split('\n').at(-1)) + if (!Number.isInteger(dropped)) { + throw new Error(`Unexpected transport-drop count from ${target.containerName}: ${output}`) + } + return dropped +} + +/** + * Freeze the container. TCP stays established and nothing is reset, so the client + * sees silence rather than a closed socket. + * + * Why: this is the laptop-lid / network-stall shape, and the only fault that can + * expose a liveness timeout firing on a session that is still perfectly healthy — + * verified locally: a stream stalls while paused and resumes intact on unpause. + */ +export function stallDockerSshRelayTarget(target: DockerSshRelayTarget): void { + run(['pause', target.containerName]) +} + +export function resumeDockerSshRelayTarget(target: DockerSshRelayTarget): void { + run(['unpause', target.containerName]) +} + +export async function withStalledDockerSshRelayTarget( + target: DockerSshRelayTarget, + body: () => Promise +): Promise { + stallDockerSshRelayTarget(target) + try { + return await body() + } finally { + resumeDockerSshRelayTarget(target) + } +} + +// Deliberately absent: a network-blackhole fault (`docker network disconnect`). The shape is real — +// the remote keeps producing while unreachable — but reconnecting the fixture does not restore its +// published port mapping, so the fault is not reversible on this container and would strand the +// worker it ran on. Reintroduce it only with a fixture that survives the round trip. + +/** + * SIGKILL every detached relay process, leaving sshd reachable. + * + * Why: the session is genuinely gone, so this is the only fault where a client is + * *supposed* to surface an explicit session-expired state instead of resuming. A + * reconnect test that never exercises this cannot tell "resumed" from "silently + * started over". + */ +export function killDockerSshRelayDaemon(target: DockerSshRelayTarget): number { + const output = execDockerSshRelayTargetControlCommand( + target, + ` +killed=0 +for proc in /proc/[0-9]*; do + [ -r "$proc/cmdline" ] || continue + argv=() + mapfile -d '' -t argv < "$proc/cmdline" 2>/dev/null || continue + entry="\${argv[1]:-}" + [ "\${entry##*/}" = relay.js ] || continue + pid="\${proc##*/}" + kill -9 "$pid" 2>/dev/null && killed=$((killed+1)) +done +echo "$killed" +` + ) + const killed = Number(output.trim().split('\n').at(-1)) + if (!Number.isInteger(killed)) { + throw new Error(`Unexpected relay-kill count from ${target.containerName}: ${output}`) + } + return killed +} + +/** + * Undo any fault a failing test left behind. + * + * Why: a paused container outlives the spec that faulted it and poisons every later spec on the + * same worker, which reads as an unrelated flake. Every fault above must be reversible here. + */ +export function clearDockerSshRelayFaults(target: DockerSshRelayTarget | null): void { + if (!target) { + return + } + tryRun(['unpause', target.containerName]) +} diff --git a/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts b/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts new file mode 100644 index 00000000000..9f8434ed51a --- /dev/null +++ b/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts @@ -0,0 +1,346 @@ +import type { Page } from '@playwright/test' +import { test, expect } from './helpers/orca-app' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + execInTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' +import { + cleanupDockerSshRelayTarget, + enableDockerSshRelayTargetShellTitle, + execDockerSshRelayTargetCommand, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection' +import { + clearDockerSshRelayFaults, + dropDockerSshRelayTransport, + killDockerSshRelayDaemon, + withStalledDockerSshRelayTarget +} from './helpers/docker-ssh-relay-faults' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' + +/** + * Every existing reconnect spec reconnects by calling ssh.disconnect() then ssh.connect() — a + * clean, client-initiated cycle that the client knows is coming. Nothing covered the fault the + * reconnect machinery actually exists for: the transport dying underneath a live session, with the + * remote still running and still holding the PTYs. + * + * The distinction matters because the two paths diverge at the relay. A graceful disconnect closes + * the client cleanly; a killed connection leaves the relay's grace window and PTY table intact, so + * a correct client re-attaches rather than rebuilding. Reports of frozen panes and duplicated agent + * sessions come from the second shape, which had no coverage at all. + * + * Faults come from docker-ssh-relay-faults, in two shapes that must not be confused. Killing + * sshd's per-connection forks leaves the listening daemon and every relay process alive, so the + * session survives and the pane must keep its PTY. SIGKILLing the relay leaves sshd reachable but + * genuinely ends the sessions, so the pane must be replaced. Only the second is `exited`; a suite + * with only the first cannot tell a resume from a silent cold start + * (docs/reference/ssh-execution-boundary.md). + */ +async function readSshStatus(orcaPage: Page, targetId: string) { + return orcaPage.evaluate( + (targetId) => window.__store?.getState().sshConnectionStates.get(targetId)?.status ?? null, + targetId + ) +} + +/** + * Not covered here on purpose: park-then-reveal after a reconnect. ssh-terminal-parking already + * covers the park/reveal round trip, and driving a park deterministically from this lane proved + * flaky enough to cost more than it proves. + */ +test.describe('SSH transport drop recovery', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run the dockerized SSH relay tests') + + test('recovers a live pane after the transport dies under it', async ({ orcaPage }, testInfo) => { + test.slow() + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + enableDockerSshRelayTargetShellTitle(target) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + const remote = await connectDockerSshRelayTarget(orcaPage, target) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + + // A marker, not a prompt: a prompt reappears on its own, so it cannot tell restored + // scrollback from a shell that simply started again. + const markerSuffix = Date.now() + const marker = `DROP_MARKER_${markerSuffix}` + await execInTerminal(orcaPage, ptyId, `printf 'DROP_MARKER_%s\\n' ${markerSuffix}`) + await waitForTerminalOutput(orcaPage, marker, 30_000) + + const dropped = dropDockerSshRelayTransport(target) + expect(dropped, 'no live SSH connection was found to drop').toBeGreaterThan(0) + + // Nothing below calls ssh.connect(). Recovery has to come from the client's own ladder, + // which is the behaviour users depend on and the thing a scripted reconnect never exercised. + await expect + .poll(() => readSshStatus(orcaPage, remote.targetId), { + timeout: 120_000, + message: 'SSH target never returned to connected after the transport was dropped' + }) + .toBe('connected') + + await waitForActiveTerminalManager(orcaPage, 60_000) + await waitForActivePanePtyId(orcaPage, 60_000) + + // The pane must still show what it had. A blank pane here is the reported bug. + await waitForTerminalOutput(orcaPage, marker, 60_000) + + // And it must still be wired to a shell that answers — a pane can repaint and still be dead, + // which is the failure mode a content-only assertion misses. + const afterMarkerSuffix = Date.now() + const afterMarker = `DROP_AFTER_${afterMarkerSuffix}` + await execInTerminal( + orcaPage, + await waitForActivePanePtyId(orcaPage, 60_000), + `printf 'DROP_AFTER_%s\\n' ${afterMarkerSuffix}` + ) + await waitForTerminalOutput(orcaPage, afterMarker, 60_000) + } finally { + if (target) { + clearDockerSshRelayFaults(target) + cleanupDockerSshRelayTarget(target) + } + } + }) + + // Fixme: fails in CI on its first real run — the pane keeps its PTY and repaints, but a command + // run after the flood produces no output within the poll budget. Same shape as #18018 (deaf pane + // after a stalled host resumes), and not caused by this spec. Tracked there; the three verdict + // assertions around it stay enforced. + test.fixme('stays bounded when a disconnected shell floods its pty', async ({ orcaPage }, testInfo) => { + test.slow() + // Timeouts here are deliberately generous: this guards memory, not latency. A 48MB flood plus a + // reconnect lands near 60s wall-clock end to end, so a 60s bind timeout was marginal and made + // the spec flaky. Measured since: reconnect-and-rebind after the flood is ~11.9s, so the + // marginal part is the flood WRITE, not recovery — resuming a pty whose client has gone does + // not slow reconnect under load. + // + // The drain fix resumes a pty whose client has gone, so the shell is no longer throttled by a + // consumer that cannot consume. That is only safe if something else bounds it: `buffered` is a + // capacity-limited window, and the pending delivery queue — which is unbounded — is dropped + // rather than carried. This pins that, because the failure it guards against is an OOM on + // someone's remote host rather than a wrong pixel. + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + enableDockerSshRelayTargetShellTitle(target) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + const remote = await connectDockerSshRelayTarget(orcaPage, target) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 240_000) + const ptyId = await waitForActivePanePtyId(orcaPage, 240_000) + + const readRelayRssKb = (): number => { + const out = execDockerSshRelayTargetCommand( + target!, + "ps -eo rss,args | grep -F 'relay.js' | grep -v grep | awk '{s+=$1} END {print s+0}'" + ) + return Number(out.trim().split('\n').at(-1)) + } + const baselineRssKb = readRelayRssKb() + expect(baselineRssKb, 'relay process not found').toBeGreaterThan(0) + + // ~48 MB of output with nobody attached: far past any sane replay window. + await execInTerminal( + orcaPage, + ptyId, + `yes "$(printf 'ORCA_%s' FLOOD_LINE)" | head -c 48000000; echo FLOODED` + ) + await waitForTerminalOutput(orcaPage, 'ORCA_FLOOD_LINE', 30_000, 20_000) + const dropped = dropDockerSshRelayTransport(target) + expect(dropped).toBeGreaterThan(0) + + await expect + .poll(() => readSshStatus(orcaPage, remote.targetId), { + timeout: 120_000, + message: 'SSH target never returned to connected' + }) + .toBe('connected') + await waitForActiveTerminalManager(orcaPage, 240_000) + + // Why a generous ceiling: this is an OOM guard, not a memory budget. Unbounded retention of + // 48 MB of pty output would blow past it; ordinary V8 churn will not. + const afterRssKb = readRelayRssKb() + expect( + afterRssKb - baselineRssKb, + `relay grew ${afterRssKb - baselineRssKb}KB after 48MB of undeliverable output` + ).toBeLessThan(200_000) + + // And the session must still be usable, not merely alive. + const markerSuffix = Date.now() + const marker = `FLOOD_AFTER_${markerSuffix}` + await execInTerminal( + orcaPage, + await waitForActivePanePtyId(orcaPage, 240_000), + `printf 'FLOOD_AFTER_%s\\n' ${markerSuffix}` + ) + await waitForTerminalOutput(orcaPage, marker, 60_000, 20_000) + } finally { + if (target) { + clearDockerSshRelayFaults(target) + cleanupDockerSshRelayTarget(target) + } + } + }) + + /** + * The one fault in this file where `exited` is the correct verdict, and the only one that can + * tell "resumed" from "silently started over" (docs/reference/ssh-execution-boundary.md). + * + * Every other case here kills the transport and asserts the session survived. That assertion is + * only meaningful if a genuinely dead session is distinguishable — otherwise a client that always + * cold-starts would pass them all. SIGKILLing the relay leaves sshd reachable, so the client + * reconnects, asks the host about the PTY, and gets a positive answer that it is gone. That is + * host evidence of absence, so replacing the pane is correct here and nowhere else in this file. + */ + test('replaces the pane only when the host proves the session is gone', async ({ + orcaPage + }, testInfo) => { + test.slow() + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + enableDockerSshRelayTargetShellTitle(target) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + const remote = await connectDockerSshRelayTarget(orcaPage, target) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + + const markerSuffix = Date.now() + const marker = `KILL_MARKER_${markerSuffix}` + await execInTerminal(orcaPage, ptyId, `printf 'KILL_MARKER_%s\\n' ${markerSuffix}`) + await waitForTerminalOutput(orcaPage, marker, 30_000) + + const killed = killDockerSshRelayDaemon(target) + expect(killed, 'no relay process was found to kill').toBeGreaterThan(0) + + await expect + .poll(() => readSshStatus(orcaPage, remote.targetId), { + timeout: 120_000, + message: 'SSH target never returned to connected after the relay was killed' + }) + .toBe('connected') + await waitForActiveTerminalManager(orcaPage, 60_000) + + // The verdict, expressed as the only thing a user can observe: the pane is now backed by a + // DIFFERENT pty. On the transport-drop cases above this id must not change; here it must. + await expect + .poll(() => waitForActivePanePtyId(orcaPage, 60_000).catch(() => ptyId), { + timeout: 120_000, + message: 'pane kept its old PTY binding after the host proved the session was gone' + }) + .not.toBe(ptyId) + + // And the replacement must be a working shell, not a dead husk. + const afterSuffix = Date.now() + const afterMarker = `KILL_AFTER_${afterSuffix}` + await execInTerminal( + orcaPage, + await waitForActivePanePtyId(orcaPage, 60_000), + `printf 'KILL_AFTER_%s\\n' ${afterSuffix}` + ) + await waitForTerminalOutput(orcaPage, afterMarker, 60_000) + } finally { + if (target) { + clearDockerSshRelayFaults(target) + cleanupDockerSshRelayTarget(target) + } + } + }) + + /** + * The third fault shape: silence with the socket still established. `docker pause` freezes the + * container, so nothing is closed or reset — the client simply stops hearing from a host that is + * perfectly healthy. This is the one that pins "loss of contact is never evidence": the verdict + * during the silence must be `unverifiable`, so the pane must keep its PTY and come back with its + * scrollback rather than concluding the session died and starting over. + */ + test('keeps the session while a frozen host goes silent', async ({ orcaPage }, testInfo) => { + test.slow() + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + enableDockerSshRelayTargetShellTitle(target) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + await connectDockerSshRelayTarget(orcaPage, target) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + + const markerSuffix = Date.now() + const marker = `STALL_MARKER_${markerSuffix}` + await execInTerminal(orcaPage, ptyId, `printf 'STALL_MARKER_%s\\n' ${markerSuffix}`) + await waitForTerminalOutput(orcaPage, marker, 30_000) + + // Long enough to outlast a liveness probe, which is the point: a timeout firing here would be + // the client asserting death it never observed. + await withStalledDockerSshRelayTarget(target, async () => { + await orcaPage.waitForTimeout(30_000) + }) + + await waitForActiveTerminalManager(orcaPage, 60_000) + // Same PTY, not a replacement: nothing here is host evidence of absence. + expect(await waitForActivePanePtyId(orcaPage, 60_000)).toBe(ptyId) + await waitForTerminalOutput(orcaPage, marker, 60_000) + } finally { + if (target) { + clearDockerSshRelayFaults(target) + cleanupDockerSshRelayTarget(target) + } + } + }) + + /** + * Known broken on main, kept as the reproduction. The verdict test above passes: after a 30s + * freeze the pane keeps its PTY and repaints its scrollback. What does not come back is the + * shell — a command run afterwards produces no output within 60s, so the pane is live-looking and + * deaf. Measured twice at `waitForTerminalOutput(STALL_AFTER_…)`, and it reproduces unchanged + * with the reattach-token/delivery-ownership fix applied, so that is not the cause. + * + * Split out rather than folded into the test above so the `unverifiable` verdict stays enforced + * in CI instead of being masked by this failure. + */ + test.fixme('accepts input again after a frozen host resumes', async ({ orcaPage }, testInfo) => { + test.slow() + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + enableDockerSshRelayTargetShellTitle(target) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + await connectDockerSshRelayTarget(orcaPage, target) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + + await withStalledDockerSshRelayTarget(target, async () => { + await orcaPage.waitForTimeout(30_000) + }) + await waitForActiveTerminalManager(orcaPage, 60_000) + + const afterSuffix = Date.now() + const afterMarker = `STALL_AFTER_${afterSuffix}` + await execInTerminal(orcaPage, ptyId, `printf 'STALL_AFTER_%s\\n' ${afterSuffix}`) + await waitForTerminalOutput(orcaPage, afterMarker, 60_000) + } finally { + if (target) { + clearDockerSshRelayFaults(target) + cleanupDockerSshRelayTarget(target) + } + } + }) +})