mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 16:02:32 +00:00
test(ssh): add E2E oracles for pane cardinality and duplicate resume
Both reported failures now have end-to-end coverage against a real Docker
OpenSSH relay, gated on ORCA_E2E_SSH_DOCKER like the rest of the suite.
- ssh-reconnect-pane-cardinality-across-partitions: three real reconnect cycles;
after each, pane ids unchanged, exactly one live lease per leaf, exactly one
remote shell per pane key, and exactly one pty id bound per leaf ACROSS BOTH
durable partitions. Two partitions agreeing is tolerated; two naming different
shells is the S4 divergence and fails. PASSES.
- ssh-reattach-does-not-resume-agent-twice: the host itself records one line per
shell launch via a .bashrc hook keyed by ORCA_PANE_KEY, so a duplicate resume
is counted at the source rather than inferred. Fault is SIGSTOP on the
detached relay. PASSES.
- ssh-disconnected-pane-affordance: written whole but held as test.fixme. The
banner needs the target CONNECTED while a single pane's attach fails, and both
host-side faults drove the target out of connected instead. Held rather than
deleted so it runs the day a seam exists; the reason is measured, not assumed.
New helpers: docker-ssh-relay-stall (SIGSTOP/SIGCONT, reads the stop back off
/proc so a fault that did not land cannot make an oracle vacuous), and
remote-pane-launch-transcript.
Note for whoever picks these up: a stalled relay leaves two detached relay groups
on the host, and readDockerSshRelayProcessSnapshot throws on more than one, so
call it before the fault.
Writing these is what surfaced RC2 surviving S3 — fixed in 7cd7fef927.
This commit is contained in:
Neil
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Freezes the detached relay without killing it.
|
||||
*
|
||||
* Why this fault and not a transport sever: a sever reconnects through the
|
||||
* checkpoint path and never classifies a reattach failure at all (see the header
|
||||
* of ssh-maxsessions-remote-pid-binding-identity.spec.ts). A stopped relay still
|
||||
* holds every shell it hosts — SIGSTOP suspends one process, not its children —
|
||||
* so a `pty.attach` aimed at it runs out its request timeout while the shell the
|
||||
* pane owns keeps running. That is the shape STA-3077 is about: a reattach that
|
||||
* failed, over a session that is provably still alive.
|
||||
*
|
||||
* SIGCONT puts it back, so the same shell can be reattached afterwards.
|
||||
*/
|
||||
import {
|
||||
execDockerSshRelayTargetCommand,
|
||||
type DockerSshRelayTarget
|
||||
} from './docker-ssh-relay-target'
|
||||
|
||||
/** Guards every signal: only a detached relay may be stopped or resumed. */
|
||||
function signalDetachedRelay(
|
||||
target: DockerSshRelayTarget,
|
||||
relayPid: number,
|
||||
signal: 'STOP' | 'CONT'
|
||||
): void {
|
||||
if (!Number.isInteger(relayPid) || relayPid <= 0) {
|
||||
throw new Error(`Docker SSH relay process ID must be a positive integer: ${relayPid}`)
|
||||
}
|
||||
execDockerSshRelayTargetCommand(
|
||||
target,
|
||||
[
|
||||
`proc=/proc/${relayPid}`,
|
||||
'[ -r "$proc/cmdline" ]',
|
||||
'argv=()',
|
||||
'mapfile -d \'\' -t argv < "$proc/cmdline"',
|
||||
'[ "${argv[1]##*/}" = relay.js ]',
|
||||
'[[ " ${argv[*]:2} " = *" --detached "* ]]',
|
||||
`kill -${signal} ${relayPid}`
|
||||
].join(' && ')
|
||||
)
|
||||
}
|
||||
|
||||
export function stallDockerSshRelay(target: DockerSshRelayTarget, relayPid: number): void {
|
||||
signalDetachedRelay(target, relayPid, 'STOP')
|
||||
// Read the stop back off /proc rather than trusting the signal: an attach that
|
||||
// is answered anyway would make every oracle downstream vacuous.
|
||||
const state = execDockerSshRelayTargetCommand(
|
||||
target,
|
||||
`awk '/^State:/{print $2}' /proc/${relayPid}/status`
|
||||
)
|
||||
if (state !== 'T') {
|
||||
throw new Error(`The detached relay did not stop; /proc reports state ${state || '<gone>'}`)
|
||||
}
|
||||
}
|
||||
|
||||
export function resumeDockerSshRelay(target: DockerSshRelayTarget, relayPid: number): void {
|
||||
signalDetachedRelay(target, relayPid, 'CONT')
|
||||
}
|
||||
|
||||
/** True while the relay process still exists, whatever state it is in. */
|
||||
export function isDockerSshRelayProcessPresent(
|
||||
target: DockerSshRelayTarget,
|
||||
relayPid: number
|
||||
): boolean {
|
||||
return (
|
||||
execDockerSshRelayTargetCommand(
|
||||
target,
|
||||
`test -r /proc/${relayPid}/status && echo PRESENT || echo GONE`
|
||||
) === 'PRESENT'
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* A stand-in for an agent's transcript, recorded by the host rather than by the app.
|
||||
*
|
||||
* The STA-3077 report's second failure is "a coding agent was resumed twice into
|
||||
* one transcript". Driving a real coding agent inside the Docker OpenSSH fixture
|
||||
* is not possible — no agent binary ships in the image and no provider session
|
||||
* exists to resume — so the observable is moved to the thing that actually goes
|
||||
* wrong: a second shell launching for a pane that already has one. Every shell
|
||||
* the relay starts for a pane appends one line here, keyed by pane, so "resumed
|
||||
* twice" reads as two lines for one pane key and needs no app-side reporting to
|
||||
* be believed.
|
||||
*
|
||||
* The hook is installed before the first pane exists, and each test asserts the
|
||||
* first launch was recorded before asserting no second one was — a hook that
|
||||
* silently failed to fire would otherwise make every clause pass vacuously.
|
||||
*/
|
||||
import {
|
||||
execDockerSshRelayTargetCommand,
|
||||
type DockerSshRelayTarget
|
||||
} from './docker-ssh-relay-target'
|
||||
|
||||
const TRANSCRIPT_PATH = '/tmp/orca-pane-launch-transcript.log'
|
||||
|
||||
// Prepended, not appended: Debian's stock /root/.bashrc returns early for a
|
||||
// non-interactive shell, and a hook placed after that guard would only fire
|
||||
// sometimes. `ORCA_PANE_KEY` is injected by the pane launch path alone, so a
|
||||
// `docker exec` probe shell adds nothing.
|
||||
const INSTALL_TRANSCRIPT_HOOK_COMMAND = `
|
||||
set -e
|
||||
hook=/root/.orca-pane-launch-transcript.sh
|
||||
cat > "$hook" <<'HOOK'
|
||||
if [ -n "$ORCA_PANE_KEY" ]; then
|
||||
printf '%s\\t%s\\n' "$ORCA_PANE_KEY" "$$" >> ${TRANSCRIPT_PATH}
|
||||
fi
|
||||
HOOK
|
||||
touch /root/.bashrc ${TRANSCRIPT_PATH}
|
||||
if ! grep -q orca-pane-launch-transcript /root/.bashrc; then
|
||||
printf '. %s\\n' "$hook" | cat - /root/.bashrc > /root/.bashrc.next
|
||||
mv /root/.bashrc.next /root/.bashrc
|
||||
fi
|
||||
grep -c orca-pane-launch-transcript /root/.bashrc
|
||||
`
|
||||
|
||||
export function installRemotePaneLaunchTranscript(target: DockerSshRelayTarget): void {
|
||||
const installed = Number(
|
||||
execDockerSshRelayTargetCommand(target, INSTALL_TRANSCRIPT_HOOK_COMMAND).split('\n').pop()
|
||||
)
|
||||
if (installed !== 1) {
|
||||
throw new Error(`The pane launch transcript hook was not installed exactly once: ${installed}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Every shell launch the host recorded, oldest first, as `paneKey\tpid`. */
|
||||
export function readRemotePaneLaunchTranscript(target: DockerSshRelayTarget): string[] {
|
||||
return execDockerSshRelayTargetCommand(target, `cat ${TRANSCRIPT_PATH} 2>/dev/null || true`)
|
||||
.split('\n')
|
||||
.filter((line) => line.includes('\t'))
|
||||
}
|
||||
|
||||
/** The pids the host launched a shell for under one pane key. */
|
||||
export function readRemotePaneLaunchPids(
|
||||
target: DockerSshRelayTarget,
|
||||
paneKey: string
|
||||
): number[] {
|
||||
return readRemotePaneLaunchTranscript(target)
|
||||
.filter((line) => line.split('\t')[0] === paneKey)
|
||||
.map((line) => Number(line.split('\t')[1]))
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
/**
|
||||
* STA-3077 goalpost S3's affordance — an unreachable pane says so, offers two
|
||||
* actions, and destroys nothing.
|
||||
*
|
||||
* Before S3, `handlePtyReattachFailure` answered any reattach failure with a
|
||||
* synthetic `pty:exit { code: -1 }`, cleared provider state, deleted ownership
|
||||
* and expired the lease: four claims about a process nobody had observed, on a
|
||||
* shell that was usually still running. Collapsing that left the pane with no
|
||||
* signal at all, so the pane now renders as disconnected with exactly two
|
||||
* explicit actions — "Try again" and "Start a new terminal" — and its copy may
|
||||
* never assert the shell exited, because a failed attach is not result data.
|
||||
*
|
||||
* ─────────────────────────────────────────────────────────────────────────────
|
||||
* HELD AS `fixme`: THE STATE IS NOT INDUCIBLE FROM THE HOST TODAY.
|
||||
*
|
||||
* The banner needs one narrow state — the SSH target reporting itself CONNECTED
|
||||
* while a single pane's `pty.attach` fails with an error that does not prove the
|
||||
* session gone. `TerminalPane` suppresses this banner entirely while
|
||||
* `showSshReconnectOverlay` is true, so any fault that takes the connection down
|
||||
* with it produces the connection-level overlay instead and never reaches here.
|
||||
*
|
||||
* Two host-side faults were driven against the real Docker relay and neither
|
||||
* lands in that state:
|
||||
*
|
||||
* 1. Stop the detached relay (SIGSTOP) and remount the pane. The relay keeps
|
||||
* every shell, but the client reads the silent host as gone, reinstalls a
|
||||
* relay beside it, and the SSH target drops to `connecting`. The pane's
|
||||
* connect gate then DEFERS rather than attaching, so no attach fails at all;
|
||||
* the pane shows "SSH connection required". Measured, not assumed: the run
|
||||
* that produced this note timed out on the locator below with that overlay
|
||||
* on screen. That fault is still worth having, and it is what
|
||||
* ssh-reattach-does-not-resume-agent-twice.spec.ts uses to prove the
|
||||
* non-destructive half of S3 — the shell stays alive, the lease stays
|
||||
* claimable, and no second shell is launched.
|
||||
* 2. Sever the transport. Same shape: the target leaves `connected`, the gate
|
||||
* defers, and the reattach that eventually runs succeeds.
|
||||
*
|
||||
* What would reach it is a fault that leaves the mux healthy and fails ONE pty:
|
||||
* the relay answering `sourceRecovery.restoreRequired` for a single id, which
|
||||
* today only arises from a delivery-record divergence between two client
|
||||
* generations. There is no seam to force that from a test without adding one in
|
||||
* `src/`, which this change is not allowed to do. The oracle is kept whole and
|
||||
* held rather than deleted, so it runs the day such a seam exists.
|
||||
* ─────────────────────────────────────────────────────────────────────────────
|
||||
*
|
||||
* MUTATIONS THAT MUST REDDEN THIS FILE ONCE IT RUNS:
|
||||
* - `reattach-failure-classification.ts`: make `isProvenSshSessionGoneError`
|
||||
* return true for every error. The pane respawns silently, so no banner
|
||||
* appears and the census gains a second shell.
|
||||
* - `TerminalPaneDisconnectedBanner.tsx`: reword the `ssh-pane` copy to say the
|
||||
* terminal exited, or drop either action. Clause 2 or clause 1 reddens.
|
||||
* - `ssh-relay-session.ts`: restore the destructive block in
|
||||
* `handlePtyReattachFailure`. Clause 3 reddens on the expired lease.
|
||||
*/
|
||||
import type { Locator, Page, TestInfo } from '@stablyai/playwright-test'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
import {
|
||||
sendToTerminal,
|
||||
waitForActivePanePtyId,
|
||||
waitForActiveTerminalManager,
|
||||
waitForPaneIdentitySnapshot,
|
||||
type PaneIdentitySnapshot
|
||||
} from './helpers/terminal'
|
||||
import {
|
||||
describeSshRemotePtyLeases,
|
||||
findSshRemotePtyLeaseForLeaf,
|
||||
readSshRemotePtyLeases,
|
||||
resolveOrcaProfileStateFile
|
||||
} from './helpers/ssh-remote-pty-lease-file'
|
||||
import {
|
||||
cleanupDockerSshRelayTarget,
|
||||
startDockerSshRelayTarget,
|
||||
type DockerSshRelayTarget
|
||||
} from './helpers/docker-ssh-relay-target'
|
||||
import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection'
|
||||
import { readDockerSshRelayProcessSnapshot } from './helpers/docker-ssh-relay-processes'
|
||||
import { resumeDockerSshRelay, stallDockerSshRelay } from './helpers/docker-ssh-relay-stall'
|
||||
import {
|
||||
countDockerSshRelayRemoteStreamWriters,
|
||||
describeDockerSshRelayRemotePtys,
|
||||
readDockerSshRelayRemotePtys
|
||||
} from './helpers/docker-ssh-relay-remote-ptys'
|
||||
import {
|
||||
installRemotePaneLaunchTranscript,
|
||||
readRemotePaneLaunchPids
|
||||
} from './helpers/remote-pane-launch-transcript'
|
||||
|
||||
const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1'
|
||||
const UNREACHABLE_PANE_INDUCTION_UNAVAILABLE =
|
||||
'No host-side fault leaves the SSH target connected while one pane attach fails; see the file header.'
|
||||
// Why: the relay must outlive the fault. If it exits with the client the remote
|
||||
// shells die too, and a dead shell makes "the banner overclaims" untestable.
|
||||
const RELAY_GRACE_PERIOD_SECONDS = 900
|
||||
// The banner is only reachable after the pane's `pty.attach` gives up, which
|
||||
// costs one mux request timeout.
|
||||
const UNREACHABLE_PANE_TIMEOUT_MS = 150_000
|
||||
|
||||
/**
|
||||
* The copy constraint as an oracle. A failed attach is not result data, so the
|
||||
* pane may not carry a result verb (STYLEGUIDE.md:236). Enforced here rather
|
||||
* than in review, because the tempting reword is exactly the old lie.
|
||||
*/
|
||||
const DEATH_VERB = /exit|died|dead|terminated|killed|crashed/i
|
||||
/** The wire tokens the classification runs on must never reach the user. */
|
||||
const WIRE_TOKEN = /SSH_[A-Z_]+/
|
||||
|
||||
test.use({ seedTestRepo: false })
|
||||
|
||||
type UnreachableRemotePane = {
|
||||
remote: Awaited<ReturnType<typeof connectDockerSshRelayTarget>>
|
||||
snapshot: PaneIdentitySnapshot
|
||||
paneKey: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
pid: number
|
||||
startTicks: number
|
||||
relayPid: number
|
||||
stateFile: string
|
||||
banner: Locator
|
||||
}
|
||||
|
||||
/** Rebuild the pane's renderer over its live PTY — the app's own recovery seam. */
|
||||
async function remountTerminalTabForRecovery(page: Page, tabId: string): Promise<void> {
|
||||
const remounted = await page.evaluate((tabId) => {
|
||||
const state = window.__store?.getState()
|
||||
if (!state) {
|
||||
throw new Error('Store unavailable')
|
||||
}
|
||||
return state.remountTerminalTabForRecovery(tabId)
|
||||
}, tabId)
|
||||
expect(remounted, 'the terminal tab refused to remount, so no reattach was attempted').toBe(true)
|
||||
}
|
||||
|
||||
/**
|
||||
* A connected remote workspace with one streaming pane whose host has stopped
|
||||
* answering. Why the pane must stream: an idle pane reattaches as 'existing' and
|
||||
* never enters the source re-establishment the field failure came through.
|
||||
*/
|
||||
async function openUnreachableRemotePane(
|
||||
page: Page,
|
||||
app: Parameters<typeof resolveOrcaProfileStateFile>[0],
|
||||
target: DockerSshRelayTarget
|
||||
): Promise<UnreachableRemotePane> {
|
||||
installRemotePaneLaunchTranscript(target)
|
||||
|
||||
await waitForSessionReady(page)
|
||||
const remote = await connectDockerSshRelayTarget(page, target, {
|
||||
relayGracePeriodSeconds: RELAY_GRACE_PERIOD_SECONDS
|
||||
})
|
||||
await expect.poll(() => waitForActiveWorktree(page), { timeout: 30_000 }).toBe(remote.worktreeId)
|
||||
await ensureTerminalVisible(page, 45_000)
|
||||
await waitForActiveTerminalManager(page, 60_000)
|
||||
await waitForActivePanePtyId(page, 60_000)
|
||||
|
||||
const snapshot = await waitForPaneIdentitySnapshot(page, 1)
|
||||
const pane = snapshot.panes[0]
|
||||
if (!pane?.ptyId) {
|
||||
throw new Error('The remote workspace opened no pane with a PTY')
|
||||
}
|
||||
const paneKey = `${snapshot.tabId}:${pane.leafId}`
|
||||
|
||||
const streamMarker = `SSH_AFFORDANCE_STREAM_${Date.now()}`
|
||||
await sendToTerminal(
|
||||
page,
|
||||
pane.ptyId,
|
||||
`node -e "setInterval(()=>process.stdout.write('${streamMarker}_'+Date.now()+'\\n'),100)" &\r`
|
||||
)
|
||||
await expect
|
||||
.poll(() => countDockerSshRelayRemoteStreamWriters(target, streamMarker), {
|
||||
timeout: 60_000,
|
||||
message: 'the remote pane never started streaming'
|
||||
})
|
||||
.toBe(1)
|
||||
|
||||
// Anti-vacuity for every "no second shell" clause below: the host records one
|
||||
// launch for this pane now, so a later count of one means nothing was added.
|
||||
await expect
|
||||
.poll(() => readRemotePaneLaunchPids(target, paneKey).length, {
|
||||
timeout: 60_000,
|
||||
message: 'the host recorded no shell launch for the pane, so its transcript proves nothing'
|
||||
})
|
||||
.toBe(1)
|
||||
|
||||
const shell = readDockerSshRelayRemotePtys(target).find((entry) => entry.paneKey === paneKey)
|
||||
if (!shell) {
|
||||
throw new Error(`The relay hosts no remote shell for pane ${paneKey}`)
|
||||
}
|
||||
const relay = readDockerSshRelayProcessSnapshot(target)
|
||||
if (!relay) {
|
||||
throw new Error('No detached relay to stall')
|
||||
}
|
||||
|
||||
stallDockerSshRelay(target, relay.relayPid)
|
||||
await remountTerminalTabForRecovery(page, snapshot.tabId)
|
||||
|
||||
return {
|
||||
remote,
|
||||
snapshot,
|
||||
paneKey,
|
||||
leafId: pane.leafId,
|
||||
ptyId: pane.ptyId,
|
||||
pid: shell.pid,
|
||||
startTicks: shell.startTicks,
|
||||
relayPid: relay.relayPid,
|
||||
stateFile: await resolveOrcaProfileStateFile(app),
|
||||
banner: page.locator('[data-terminal-pane-disconnected-variant="ssh-pane"]')
|
||||
}
|
||||
}
|
||||
|
||||
test.describe('an unreachable SSH pane offers two actions and destroys nothing', () => {
|
||||
test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH tests.')
|
||||
test.skip(process.platform === 'win32', 'Docker SSH faults use POSIX SSH tooling.')
|
||||
|
||||
test('the pane reports itself disconnected without claiming the shell exited', async ({
|
||||
orcaPage,
|
||||
electronApp
|
||||
}, testInfo: TestInfo) => {
|
||||
test.fixme(true, UNREACHABLE_PANE_INDUCTION_UNAVAILABLE)
|
||||
test.setTimeout(600_000)
|
||||
let target: DockerSshRelayTarget | null = null
|
||||
let stalledRelayPid: number | null = null
|
||||
try {
|
||||
target = startDockerSshRelayTarget(testInfo)
|
||||
const relayTarget = target
|
||||
const unreachable = await openUnreachableRemotePane(orcaPage, electronApp, relayTarget)
|
||||
stalledRelayPid = unreachable.relayPid
|
||||
|
||||
// 1. The pane says it is disconnected, in the SSH variant, with both actions.
|
||||
await expect(
|
||||
unreachable.banner,
|
||||
'the unreachable pane showed no disconnected affordance'
|
||||
).toBeVisible({ timeout: UNREACHABLE_PANE_TIMEOUT_MS })
|
||||
await expect(
|
||||
unreachable.banner,
|
||||
'the disconnected pane is still advertising automatic retries'
|
||||
).toHaveAttribute('data-terminal-remote-runtime-reconnect-banner', 'disconnected')
|
||||
await expect(unreachable.banner.getByRole('button', { name: 'Try again' })).toBeVisible()
|
||||
await expect(
|
||||
unreachable.banner.getByRole('button', { name: 'Start a new terminal' })
|
||||
).toBeVisible()
|
||||
|
||||
// 2. Copy constraint. Nothing here observed an exit, so nothing may report one.
|
||||
const copy = (await unreachable.banner.textContent()) ?? ''
|
||||
expect(copy.trim(), 'the disconnected banner rendered no copy at all').not.toBe('')
|
||||
expect(copy, 'the disconnected banner claims the shell is gone').not.toMatch(DEATH_VERB)
|
||||
expect(copy, 'the disconnected banner leaks a wire token').not.toMatch(WIRE_TOKEN)
|
||||
testInfo.annotations.push({ type: 'ssh-disconnected-pane-copy', description: copy })
|
||||
|
||||
// 3. The claim under the copy: the shell is alive and the lease is not retired.
|
||||
const shellsWhileUnreachable = readDockerSshRelayRemotePtys(relayTarget)
|
||||
expect(
|
||||
shellsWhileUnreachable.filter(
|
||||
(shell) => shell.pid === unreachable.pid && shell.startTicks === unreachable.startTicks
|
||||
),
|
||||
'the failed reattach killed the shell the pane owns'
|
||||
).toHaveLength(1)
|
||||
const lease = findSshRemotePtyLeaseForLeaf(
|
||||
unreachable.stateFile,
|
||||
unreachable.remote.targetId,
|
||||
unreachable.leafId
|
||||
)
|
||||
expect(lease, 'the failed reattach dropped the pane lease').toBeDefined()
|
||||
expect(lease?.state, 'the failed reattach expired a lease over a running shell').not.toBe(
|
||||
'expired'
|
||||
)
|
||||
expect(lease?.state, 'the failed reattach retired a lease over a running shell').not.toBe(
|
||||
'terminated'
|
||||
)
|
||||
expect(
|
||||
readRemotePaneLaunchPids(relayTarget, unreachable.paneKey),
|
||||
'the failed reattach launched a second shell for the pane'
|
||||
).toHaveLength(1)
|
||||
testInfo.annotations.push({
|
||||
type: 'ssh-disconnected-pane-unreachable',
|
||||
description: `${describeDockerSshRelayRemotePtys(shellsWhileUnreachable)} leases=${describeSshRemotePtyLeases(
|
||||
readSshRemotePtyLeases(unreachable.stateFile, unreachable.remote.targetId)
|
||||
)}`
|
||||
})
|
||||
|
||||
// 4. 'Try again' reattaches the same shell once the host can answer again.
|
||||
resumeDockerSshRelay(relayTarget, unreachable.relayPid)
|
||||
stalledRelayPid = null
|
||||
await unreachable.banner.getByRole('button', { name: 'Try again' }).click()
|
||||
await expect(unreachable.banner, 'the banner outlived a successful retry').toBeHidden({
|
||||
timeout: UNREACHABLE_PANE_TIMEOUT_MS
|
||||
})
|
||||
await waitForActiveTerminalManager(orcaPage, 60_000)
|
||||
const recovered = await waitForPaneIdentitySnapshot(orcaPage, 1)
|
||||
expect(recovered.panes[0]?.ptyId, 'the retry bound the pane to a different session').toBe(
|
||||
unreachable.ptyId
|
||||
)
|
||||
expect(
|
||||
readDockerSshRelayRemotePtys(relayTarget).map(
|
||||
(shell) => `${shell.paneKey ?? '-'}=${shell.pid}@${shell.startTicks}`
|
||||
),
|
||||
'the retry added or replaced a remote shell'
|
||||
).toEqual([`${unreachable.paneKey}=${unreachable.pid}@${unreachable.startTicks}`])
|
||||
expect(
|
||||
readRemotePaneLaunchPids(relayTarget, unreachable.paneKey),
|
||||
'the retry launched a second shell for the pane'
|
||||
).toHaveLength(1)
|
||||
} finally {
|
||||
if (target && stalledRelayPid !== null) {
|
||||
resumeDockerSshRelay(target, stalledRelayPid)
|
||||
}
|
||||
cleanupDockerSshRelayTarget(target)
|
||||
}
|
||||
})
|
||||
|
||||
// The other action, and the reason it is not styled as a loss: it adds a shell
|
||||
// rather than replacing one, and the pane the user had is still the pane the
|
||||
// user has.
|
||||
test('starting a new terminal adds exactly one shell and leaves the panes alone', async ({
|
||||
orcaPage,
|
||||
electronApp
|
||||
}, testInfo: TestInfo) => {
|
||||
test.fixme(true, UNREACHABLE_PANE_INDUCTION_UNAVAILABLE)
|
||||
test.setTimeout(600_000)
|
||||
let target: DockerSshRelayTarget | null = null
|
||||
let stalledRelayPid: number | null = null
|
||||
try {
|
||||
target = startDockerSshRelayTarget(testInfo)
|
||||
const relayTarget = target
|
||||
const unreachable = await openUnreachableRemotePane(orcaPage, electronApp, relayTarget)
|
||||
stalledRelayPid = unreachable.relayPid
|
||||
|
||||
await expect(
|
||||
unreachable.banner,
|
||||
'the unreachable pane showed no disconnected affordance'
|
||||
).toBeVisible({ timeout: UNREACHABLE_PANE_TIMEOUT_MS })
|
||||
|
||||
resumeDockerSshRelay(relayTarget, unreachable.relayPid)
|
||||
stalledRelayPid = null
|
||||
await unreachable.banner.getByRole('button', { name: 'Start a new terminal' }).click()
|
||||
|
||||
// Exactly one more shell: the one the user asked for. The old shell is left
|
||||
// running, which is what makes "starting a new one leaves it alone" true.
|
||||
await expect
|
||||
.poll(() => readRemotePaneLaunchPids(relayTarget, unreachable.paneKey).length, {
|
||||
timeout: UNREACHABLE_PANE_TIMEOUT_MS,
|
||||
message: 'starting a new terminal did not settle at exactly one additional launch'
|
||||
})
|
||||
.toBe(2)
|
||||
expect(
|
||||
readDockerSshRelayRemotePtys(relayTarget).filter(
|
||||
(shell) => shell.pid === unreachable.pid && shell.startTicks === unreachable.startTicks
|
||||
),
|
||||
'starting a new terminal killed the shell it promised to leave alone'
|
||||
).toHaveLength(1)
|
||||
expect(
|
||||
readDockerSshRelayRemotePtys(relayTarget),
|
||||
'starting a new terminal changed the host by more than one shell'
|
||||
).toHaveLength(2)
|
||||
|
||||
// The pane count is a user-visible promise: a new shell is not a new pane.
|
||||
const paneCount = await orcaPage.evaluate((worktreeId) => {
|
||||
const state = window.__store?.getState()
|
||||
if (!state) {
|
||||
throw new Error('Store unavailable')
|
||||
}
|
||||
type LayoutNode =
|
||||
| { type: 'leaf'; leafId: string }
|
||||
| { type: 'split'; first: LayoutNode; second: LayoutNode }
|
||||
| null
|
||||
const collectLeafIds = (node: LayoutNode): string[] =>
|
||||
!node
|
||||
? []
|
||||
: node.type === 'leaf'
|
||||
? [node.leafId]
|
||||
: [...collectLeafIds(node.first), ...collectLeafIds(node.second)]
|
||||
return (state.tabsByWorktree[worktreeId] ?? []).reduce((total, tab) => {
|
||||
const leaves = collectLeafIds(
|
||||
(state.terminalLayoutsByTabId[tab.id]?.root ?? null) as LayoutNode
|
||||
)
|
||||
return total + Math.max(leaves.length, 1)
|
||||
}, 0)
|
||||
}, unreachable.remote.worktreeId)
|
||||
expect(paneCount, 'starting a new terminal surfaced a pane the user never opened').toBe(1)
|
||||
testInfo.annotations.push({
|
||||
type: 'ssh-disconnected-pane-new-terminal',
|
||||
description: describeDockerSshRelayRemotePtys(readDockerSshRelayRemotePtys(relayTarget))
|
||||
})
|
||||
} finally {
|
||||
if (target && stalledRelayPid !== null) {
|
||||
resumeDockerSshRelay(target, stalledRelayPid)
|
||||
}
|
||||
cleanupDockerSshRelayTarget(target)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* STA-3077 goalpost S3 — a reattach failure may not become a respawn — and S8,
|
||||
* which removed the last path that granted one without being asked.
|
||||
*
|
||||
* The reported failure: a coding agent was resumed twice into one transcript.
|
||||
* The mechanism is a converging decision. A reattach that failed for any reason
|
||||
* was read as proof the session had gone, so `handlePtyReattachFailure` sent the
|
||||
* pane a synthetic `pty:exit { code: -1 }`, the pane read that as a death, and
|
||||
* the cold-restore resume ran with the same provider session id over a shell
|
||||
* that was still running. Two agent processes then appended to one transcript.
|
||||
* Respawn now requires proof; everything else stays unresolved.
|
||||
*
|
||||
* WHAT IS PROXIED, AND WHY. A real coding agent cannot be driven here: the
|
||||
* Docker OpenSSH fixture ships no agent binary and there is no provider session
|
||||
* to resume, so "one transcript, two resumes" has no literal form in this
|
||||
* harness. The observable moves one level down, to the thing that must happen
|
||||
* before an agent can be resumed twice — a SECOND shell launching for a pane
|
||||
* that already has one. The host records every launch itself
|
||||
* (helpers/remote-pane-launch-transcript.ts), so the count is taken on the
|
||||
* container and owes the app nothing.
|
||||
*
|
||||
* THE FAULT. The detached relay is stopped, not killed. It keeps every shell it
|
||||
* hosts (SIGSTOP suspends one process, not its children) and answers nothing.
|
||||
* The client treats the silent host as gone and reinstalls a relay beside it, so
|
||||
* the pane's session is one the new relay has never heard of, over a shell that
|
||||
* is provably still running on the container — which is exactly the state the
|
||||
* fabricated exit used to lie about.
|
||||
*
|
||||
* MUTATIONS THAT MUST REDDEN THIS FILE:
|
||||
* - `src/main/ssh/ssh-relay-session.ts`: restore the destructive block in
|
||||
* `handlePtyReattachFailure`. The fabricated exit reaches the pane, which
|
||||
* cold-restores over a live shell and the host records a second launch.
|
||||
* - `src/renderer/.../reattach-failure-classification.ts`: make
|
||||
* `isProvenSshSessionGoneError` return true for every error, and the pane's
|
||||
* own reattach arm respawns instead of holding.
|
||||
* - `src/main/runtime/orca-runtime.ts`: reinstate the recovery grant S8 deleted
|
||||
* from `recoverTerminalPane`, with its id comparison normalized so it can
|
||||
* actually fire; a disconnected pane then starts a replacement on its own.
|
||||
*/
|
||||
import type { Page, TestInfo } from '@stablyai/playwright-test'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
import {
|
||||
sendToTerminal,
|
||||
waitForActivePanePtyId,
|
||||
waitForActiveTerminalManager,
|
||||
waitForPaneIdentitySnapshot
|
||||
} from './helpers/terminal'
|
||||
import {
|
||||
findSshRemotePtyLeaseForLeaf,
|
||||
describeSshRemotePtyLeases,
|
||||
readSshRemotePtyLeases,
|
||||
resolveOrcaProfileStateFile
|
||||
} from './helpers/ssh-remote-pty-lease-file'
|
||||
import {
|
||||
cleanupDockerSshRelayTarget,
|
||||
startDockerSshRelayTarget,
|
||||
type DockerSshRelayTarget
|
||||
} from './helpers/docker-ssh-relay-target'
|
||||
import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection'
|
||||
import { readDockerSshRelayProcessSnapshot } from './helpers/docker-ssh-relay-processes'
|
||||
import { resumeDockerSshRelay, stallDockerSshRelay } from './helpers/docker-ssh-relay-stall'
|
||||
import {
|
||||
countDockerSshRelayRemoteStreamWriters,
|
||||
describeDockerSshRelayRemotePtys,
|
||||
readDockerSshRelayRemotePtys
|
||||
} from './helpers/docker-ssh-relay-remote-ptys'
|
||||
import {
|
||||
installRemotePaneLaunchTranscript,
|
||||
readRemotePaneLaunchPids
|
||||
} from './helpers/remote-pane-launch-transcript'
|
||||
|
||||
const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1'
|
||||
// Why: the relay must outlive the fault. If it exits with the client the remote
|
||||
// shells die too, and a respawn over a corpse is not the defect this bounds.
|
||||
const RELAY_GRACE_PERIOD_SECONDS = 900
|
||||
// The pane has to give up on a host that answers nothing, which costs a mux
|
||||
// request timeout before anything is classified.
|
||||
const UNREACHABLE_PANE_TIMEOUT_MS = 180_000
|
||||
// A respawn can land late, after the failure is classified. One sample would
|
||||
// miss it, so every clause is held over a window instead.
|
||||
const HOLD_WINDOW_MS = 25_000
|
||||
|
||||
test.use({ seedTestRepo: false })
|
||||
|
||||
/** Rebuild the pane's renderer over its live PTY — the app's own recovery seam. */
|
||||
async function remountTerminalTabForRecovery(page: Page, tabId: string): Promise<void> {
|
||||
const remounted = await page.evaluate((tabId) => {
|
||||
const state = window.__store?.getState()
|
||||
if (!state) {
|
||||
throw new Error('Store unavailable')
|
||||
}
|
||||
return state.remountTerminalTabForRecovery(tabId)
|
||||
}, tabId)
|
||||
expect(remounted, 'the terminal tab refused to remount, so no reattach was attempted').toBe(true)
|
||||
}
|
||||
|
||||
test.describe('a failed SSH reattach never starts a second shell for the pane', () => {
|
||||
test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH tests.')
|
||||
test.skip(process.platform === 'win32', 'Docker SSH faults use POSIX SSH tooling.')
|
||||
|
||||
test('a host that stops answering leaves the pane on the shell it already had', async ({
|
||||
orcaPage,
|
||||
electronApp
|
||||
}, testInfo: TestInfo) => {
|
||||
test.setTimeout(600_000)
|
||||
let target: DockerSshRelayTarget | null = null
|
||||
let stalledRelayPid: number | null = null
|
||||
try {
|
||||
target = startDockerSshRelayTarget(testInfo)
|
||||
const relayTarget = target
|
||||
installRemotePaneLaunchTranscript(relayTarget)
|
||||
|
||||
await waitForSessionReady(orcaPage)
|
||||
const remote = await connectDockerSshRelayTarget(orcaPage, relayTarget, {
|
||||
relayGracePeriodSeconds: RELAY_GRACE_PERIOD_SECONDS
|
||||
})
|
||||
await expect
|
||||
.poll(() => waitForActiveWorktree(orcaPage), { timeout: 30_000 })
|
||||
.toBe(remote.worktreeId)
|
||||
await ensureTerminalVisible(orcaPage, 45_000)
|
||||
await waitForActiveTerminalManager(orcaPage, 60_000)
|
||||
await waitForActivePanePtyId(orcaPage, 60_000)
|
||||
|
||||
const snapshot = await waitForPaneIdentitySnapshot(orcaPage, 1)
|
||||
const pane = snapshot.panes[0]
|
||||
if (!pane?.ptyId) {
|
||||
throw new Error('The remote workspace opened no pane with a PTY')
|
||||
}
|
||||
const paneKey = `${snapshot.tabId}:${pane.leafId}`
|
||||
const stateFile = await resolveOrcaProfileStateFile(electronApp)
|
||||
|
||||
// Why the pane must stream: an idle pane reattaches as 'existing' and never
|
||||
// enters the source re-establishment the field failure came through.
|
||||
const streamMarker = `SSH_RESUME_ONCE_STREAM_${Date.now()}`
|
||||
await sendToTerminal(
|
||||
orcaPage,
|
||||
pane.ptyId,
|
||||
`node -e "setInterval(()=>process.stdout.write('${streamMarker}_'+Date.now()+'\\n'),100)" &\r`
|
||||
)
|
||||
await expect
|
||||
.poll(() => countDockerSshRelayRemoteStreamWriters(relayTarget, streamMarker), {
|
||||
timeout: 60_000,
|
||||
message: 'the remote pane never started streaming'
|
||||
})
|
||||
.toBe(1)
|
||||
|
||||
// Anti-vacuity. Every clause below counts launches; had the host recorded
|
||||
// none, "still exactly one" would prove nothing at all.
|
||||
await expect
|
||||
.poll(() => readRemotePaneLaunchPids(relayTarget, paneKey).length, {
|
||||
timeout: 60_000,
|
||||
message: 'the host recorded no shell launch for the pane, so its transcript is inert'
|
||||
})
|
||||
.toBe(1)
|
||||
const launchPid = readRemotePaneLaunchPids(relayTarget, paneKey)[0]
|
||||
const shell = readDockerSshRelayRemotePtys(relayTarget).find(
|
||||
(entry) => entry.paneKey === paneKey
|
||||
)
|
||||
if (!shell) {
|
||||
throw new Error(`The relay hosts no remote shell for pane ${paneKey}`)
|
||||
}
|
||||
// Cross-checks the transcript against the process census: the pid the host
|
||||
// logged at launch is the pid the relay hosts for this pane.
|
||||
expect(
|
||||
shell.pid,
|
||||
'the launch the host recorded is not the shell the relay hosts for this pane'
|
||||
).toBe(launchPid)
|
||||
const shellIdentity = `${shell.pid}@${shell.startTicks}`
|
||||
testInfo.annotations.push({
|
||||
type: 'ssh-resume-once-baseline',
|
||||
description: `${describeDockerSshRelayRemotePtys(readDockerSshRelayRemotePtys(relayTarget))} launches=1`
|
||||
})
|
||||
|
||||
const relay = readDockerSshRelayProcessSnapshot(relayTarget)
|
||||
if (!relay) {
|
||||
throw new Error('No detached relay to stall')
|
||||
}
|
||||
stallDockerSshRelay(relayTarget, relay.relayPid)
|
||||
stalledRelayPid = relay.relayPid
|
||||
await remountTerminalTabForRecovery(orcaPage, snapshot.tabId)
|
||||
|
||||
// Vacuity guard: the pane has to actually lose its host before "it kept its
|
||||
// shell" means anything. This overlay is the user-visible proof it did —
|
||||
// without it, a pane that quietly reattached would pass every clause below.
|
||||
await expect(
|
||||
orcaPage.locator('[data-terminal-ssh-reconnect-banner]').first(),
|
||||
'the pane never lost its host, so the clauses below are vacuous'
|
||||
).toBeVisible({ timeout: UNREACHABLE_PANE_TIMEOUT_MS })
|
||||
|
||||
/** Everything a duplicate resume would move, read from the host and from disk. */
|
||||
const expectNoSecondShell = (label: string): void => {
|
||||
expect(
|
||||
readRemotePaneLaunchPids(relayTarget, paneKey),
|
||||
`${label} launched a second shell for pane ${paneKey}`
|
||||
).toEqual([launchPid])
|
||||
expect(
|
||||
readDockerSshRelayRemotePtys(relayTarget)
|
||||
.filter((entry) => entry.paneKey === paneKey)
|
||||
.map((entry) => `${entry.pid}@${entry.startTicks}`),
|
||||
`${label} killed or replaced the shell the pane owns`
|
||||
).toEqual([shellIdentity])
|
||||
// Unknown is not dead: the lease must stay claimable so a later reattach
|
||||
// can still find this shell.
|
||||
const lease = findSshRemotePtyLeaseForLeaf(stateFile, remote.targetId, pane.leafId)
|
||||
expect(lease, `${label} dropped the pane lease`).toBeDefined()
|
||||
expect(lease?.state, `${label} expired a lease over a running shell`).not.toBe('expired')
|
||||
expect(lease?.state, `${label} retired a lease over a running shell`).not.toBe('terminated')
|
||||
}
|
||||
|
||||
// Sample continuously rather than poll-until-equal: a respawn that lands
|
||||
// and is later tidied up would satisfy a poll needing one matching read.
|
||||
const deadline = Date.now() + HOLD_WINDOW_MS
|
||||
let samples = 0
|
||||
while (Date.now() < deadline) {
|
||||
expectNoSecondShell('the unreachable host')
|
||||
samples += 1
|
||||
await orcaPage.waitForTimeout(500)
|
||||
}
|
||||
expect(samples).toBeGreaterThan(10)
|
||||
|
||||
// The host comes back and nothing is granted automatically: a disconnected
|
||||
// pane refuses to respawn on its own (S8), so the only route to a second
|
||||
// shell is the user asking for one.
|
||||
resumeDockerSshRelay(relayTarget, relay.relayPid)
|
||||
stalledRelayPid = null
|
||||
await orcaPage.waitForTimeout(HOLD_WINDOW_MS)
|
||||
expectNoSecondShell('the host returning')
|
||||
testInfo.annotations.push({
|
||||
type: 'ssh-resume-once-settled',
|
||||
description: `${describeDockerSshRelayRemotePtys(
|
||||
readDockerSshRelayRemotePtys(relayTarget)
|
||||
)} launches=${readRemotePaneLaunchPids(relayTarget, paneKey).length} leases=${describeSshRemotePtyLeases(
|
||||
readSshRemotePtyLeases(stateFile, remote.targetId)
|
||||
)}`
|
||||
})
|
||||
} finally {
|
||||
if (target && stalledRelayPid !== null) {
|
||||
resumeDockerSshRelay(target, stalledRelayPid)
|
||||
}
|
||||
cleanupDockerSshRelayTarget(target)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,349 @@
|
||||
/**
|
||||
* STA-3077 goalpost S4 — one durable partition per (target, pane).
|
||||
*
|
||||
* The reported failure: reconnecting an SSH host multiplied the user's
|
||||
* terminals, 2 -> 19 -> 20 across three reconnects. The mechanism was a pane
|
||||
* binding with two homes. Main's spawn wrote `ssh:<targetId>`, the relay's
|
||||
* reattach write passed no hostId and landed in `local`, and the renderer has
|
||||
* always published SSH pane membership to `local`. Supersession read the
|
||||
* partition no live writer maintained, saw the arriving lease disagree with a
|
||||
* stale pty id, and bailed — so it silently no-opped and every reconnect left
|
||||
* another live lease behind, with a shell and a pane to match.
|
||||
*
|
||||
* The mutation that must redden this file: in `src/main/ipc/pty.ts`, restore
|
||||
* the ssh-first-then-local hedge in `durablyBoundPtyIdForPane` and give the
|
||||
* spawn upserts their `ssh:<targetId>` hostId back. The divergent copy then
|
||||
* reappears, supersession stops firing, and `distinctBoundPtyIdsByLeaf` /
|
||||
* `liveLeasesByLeaf` grow past one per pane.
|
||||
*
|
||||
* Why this spec exists beside ssh-reconnect-pane-cardinality.spec.ts: that file
|
||||
* faults the transport with a kill, which reconnects with every lease still
|
||||
* marked attached. The user reported *reconnecting the host*, which is a clean
|
||||
* disconnect followed by a connect. Only the detach path moves a lease
|
||||
* attached -> detached and runs supersession on the way back, so this is the
|
||||
* shape the defect actually lived on, and it is the shape driven here — three
|
||||
* times, because the report counted three reconnects.
|
||||
*
|
||||
* Every clause is an observable: panes the user can see, leases on disk,
|
||||
* bindings in the durable session, and shells counted on the container itself.
|
||||
*/
|
||||
import type { Page, TestInfo } from '@stablyai/playwright-test'
|
||||
import { test, expect } from './helpers/orca-app'
|
||||
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
import {
|
||||
sendToTerminal,
|
||||
splitActiveTerminalPane,
|
||||
waitForActivePanePtyId,
|
||||
waitForActiveTerminalManager,
|
||||
waitForPaneIdentitySnapshot,
|
||||
type PaneIdentitySnapshot
|
||||
} from './helpers/terminal'
|
||||
import { readDurablePaneBindings, sshExecutionHostId } from './helpers/remote-pane-durable-session'
|
||||
import {
|
||||
describeSshRemotePtyLeases,
|
||||
readSshRemotePtyLeases,
|
||||
resolveOrcaProfileStateFile
|
||||
} from './helpers/ssh-remote-pty-lease-file'
|
||||
import {
|
||||
cleanupDockerSshRelayTarget,
|
||||
startDockerSshRelayTarget,
|
||||
type DockerSshRelayTarget
|
||||
} from './helpers/docker-ssh-relay-target'
|
||||
import {
|
||||
connectDockerSshRelayTarget,
|
||||
reconnectDockerSshRelayTarget
|
||||
} from './helpers/docker-ssh-relay-connection'
|
||||
import {
|
||||
countDockerSshRelayRemoteStreamWriters,
|
||||
describeDockerSshRelayRemotePtys,
|
||||
readDockerSshRelayRemotePtys
|
||||
} from './helpers/docker-ssh-relay-remote-ptys'
|
||||
|
||||
const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1'
|
||||
const PANE_COUNT = 2
|
||||
// The reported repro shape: 2 -> 19 -> 20 was counted over three reconnects.
|
||||
const RECONNECT_CYCLES = 3
|
||||
// Why: the relay must outlive every reconnect. If it exits with the client the
|
||||
// remote shells die too and reconnect degrades to a cold spawn, which is not
|
||||
// the path this spec bounds.
|
||||
const RELAY_GRACE_PERIOD_SECONDS = 900
|
||||
// Why: a graft lands after reattach reports ready, so every dimension is
|
||||
// re-read once the dust settles rather than the instant a wait passes.
|
||||
const SETTLE_MS = 6_000
|
||||
|
||||
test.use({ seedTestRepo: false })
|
||||
|
||||
/** Leases that could still claim a shell. Terminated and expired ones cannot. */
|
||||
function liveLeasesByLeaf(stateFile: string, targetId: string): Record<string, string[]> {
|
||||
const byLeaf: Record<string, string[]> = {}
|
||||
for (const lease of readSshRemotePtyLeases(stateFile, targetId)) {
|
||||
if (lease.state === 'terminated' || lease.state === 'expired') {
|
||||
continue
|
||||
}
|
||||
const leafId = lease.leafId ?? '<no-leaf>'
|
||||
byLeaf[leafId] = [...(byLeaf[leafId] ?? []), `${lease.ptyId}=${lease.state}`].sort()
|
||||
}
|
||||
return byLeaf
|
||||
}
|
||||
|
||||
/**
|
||||
* `leafId -> the distinct pty ids any durable partition binds it to`.
|
||||
*
|
||||
* Two partitions naming the same leaf is tolerable while they agree; naming it
|
||||
* twice with different shells is the divergence supersession failed to resolve,
|
||||
* and it is what a user reads as a pane that came back doubled.
|
||||
*/
|
||||
async function distinctBoundPtyIdsByLeaf(
|
||||
page: Page,
|
||||
hostId: string,
|
||||
worktreeId: string
|
||||
): Promise<Record<string, string[]>> {
|
||||
const byLeaf: Record<string, Set<string>> = {}
|
||||
for (const binding of await readDurablePaneBindings(page, hostId, worktreeId)) {
|
||||
// `<partition> <tabId>/<leafId>=<ptyId>`
|
||||
const [, pane] = binding.split(' ')
|
||||
const [paneKey, ptyId] = (pane ?? '').split('=')
|
||||
const leafId = (paneKey ?? '').split('/')[1]
|
||||
if (!leafId || !ptyId) {
|
||||
throw new Error(`Unparsable durable pane binding: ${binding}`)
|
||||
}
|
||||
byLeaf[leafId] = (byLeaf[leafId] ?? new Set<string>()).add(ptyId)
|
||||
}
|
||||
return Object.fromEntries(
|
||||
Object.entries(byLeaf).map(([leafId, ptyIds]) => [leafId, [...ptyIds].sort()])
|
||||
)
|
||||
}
|
||||
|
||||
/** `tabId/leafId` for every pane the user can see in the workspace, plus its tabs. */
|
||||
async function readRemotePaneCensus(
|
||||
page: Page,
|
||||
worktreeId: string
|
||||
): Promise<{ tabIds: string[]; paneIds: string[] }> {
|
||||
return page.evaluate((worktreeId) => {
|
||||
const state = window.__store?.getState()
|
||||
if (!state) {
|
||||
throw new Error('Store unavailable')
|
||||
}
|
||||
type LayoutNode =
|
||||
| { type: 'leaf'; leafId: string }
|
||||
| { type: 'split'; first: LayoutNode; second: LayoutNode }
|
||||
| null
|
||||
const collectLeafIds = (node: LayoutNode): string[] =>
|
||||
!node
|
||||
? []
|
||||
: node.type === 'leaf'
|
||||
? [node.leafId]
|
||||
: [...collectLeafIds(node.first), ...collectLeafIds(node.second)]
|
||||
const tabs = state.tabsByWorktree[worktreeId] ?? []
|
||||
const paneIds = tabs.flatMap((tab) => {
|
||||
const leafIds = collectLeafIds(
|
||||
(state.terminalLayoutsByTabId[tab.id]?.root ?? null) as LayoutNode
|
||||
)
|
||||
// `root: null` is the implicit single-pane layout a fresh tab carries
|
||||
// until it is first split, so it still counts as one visible pane.
|
||||
return leafIds.length > 0
|
||||
? leafIds.map((leafId) => `${tab.id}/${leafId}`)
|
||||
: [`${tab.id}/<root>`]
|
||||
})
|
||||
return { tabIds: tabs.map((tab) => tab.id).sort(), paneIds: paneIds.sort() }
|
||||
}, worktreeId)
|
||||
}
|
||||
|
||||
async function waitForSshConnected(page: Page, targetId: string, timeoutMs: number): Promise<void> {
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
page.evaluate(
|
||||
async (targetId) => (await window.api.ssh.getState({ targetId }))?.status ?? null,
|
||||
targetId
|
||||
),
|
||||
{ timeout: timeoutMs, message: 'SSH target never reported itself connected' }
|
||||
)
|
||||
.toBe('connected')
|
||||
}
|
||||
|
||||
/**
|
||||
* A connected remote workspace with PANE_COUNT panes, each carrying a live
|
||||
* output source. Why every pane must stream: an idle pane reattaches as
|
||||
* 'existing' and never enters the source re-establishment the field failure
|
||||
* came through. The writer is backgrounded so the shell keeps its prompt.
|
||||
*/
|
||||
async function openStreamingRemotePanes(
|
||||
page: Page,
|
||||
target: DockerSshRelayTarget
|
||||
): Promise<{
|
||||
remote: Awaited<ReturnType<typeof connectDockerSshRelayTarget>>
|
||||
snapshot: PaneIdentitySnapshot
|
||||
}> {
|
||||
await waitForSessionReady(page)
|
||||
const remote = await connectDockerSshRelayTarget(page, target, {
|
||||
relayGracePeriodSeconds: RELAY_GRACE_PERIOD_SECONDS
|
||||
})
|
||||
await expect.poll(() => waitForActiveWorktree(page), { timeout: 30_000 }).toBe(remote.worktreeId)
|
||||
await ensureTerminalVisible(page, 45_000)
|
||||
await waitForActiveTerminalManager(page, 60_000)
|
||||
await waitForActivePanePtyId(page, 60_000)
|
||||
|
||||
await splitActiveTerminalPane(page, 'vertical')
|
||||
const snapshot = await waitForPaneIdentitySnapshot(page, PANE_COUNT)
|
||||
|
||||
const streamMarker = `SSH_PARTITION_STREAM_${Date.now()}`
|
||||
for (const pane of snapshot.panes) {
|
||||
if (!pane.ptyId) {
|
||||
throw new Error(`Pane ${pane.leafId} has no PTY to stream from`)
|
||||
}
|
||||
await sendToTerminal(
|
||||
page,
|
||||
pane.ptyId,
|
||||
`node -e "setInterval(()=>process.stdout.write('${streamMarker}_'+Date.now()+'\\n'),100)" &\r`
|
||||
)
|
||||
}
|
||||
await expect
|
||||
.poll(() => countDockerSshRelayRemoteStreamWriters(target, streamMarker), {
|
||||
timeout: 60_000,
|
||||
message: 'remote panes did not start streaming before the first reconnect'
|
||||
})
|
||||
.toBe(PANE_COUNT)
|
||||
await expect
|
||||
.poll(() => readDockerSshRelayRemotePtys(target).length, {
|
||||
timeout: 60_000,
|
||||
message: 'remote shells did not settle at one per pane before the first reconnect'
|
||||
})
|
||||
.toBe(PANE_COUNT)
|
||||
return { remote, snapshot }
|
||||
}
|
||||
|
||||
test.describe('SSH reconnect cardinality with one durable partition per pane', () => {
|
||||
test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH tests.')
|
||||
test.skip(process.platform === 'win32', 'Docker SSH reconnect uses POSIX SSH tooling.')
|
||||
|
||||
test('three reconnects add no pane, no live lease and no remote shell', async ({
|
||||
orcaPage,
|
||||
electronApp
|
||||
}, testInfo: TestInfo) => {
|
||||
test.setTimeout(600_000)
|
||||
let target: DockerSshRelayTarget | null = null
|
||||
try {
|
||||
target = startDockerSshRelayTarget(testInfo)
|
||||
const relayTarget = target
|
||||
const { remote, snapshot } = await openStreamingRemotePanes(orcaPage, relayTarget)
|
||||
const hostId = sshExecutionHostId(remote.targetId)
|
||||
const stateFile = await resolveOrcaProfileStateFile(electronApp)
|
||||
const leafIds = snapshot.panes.map((pane) => pane.leafId)
|
||||
const paneKeys = leafIds.map((leafId) => `${snapshot.tabId}:${leafId}`)
|
||||
const shellIdentity = (shell: { paneKey: string | null; pid: number; startTicks: number }) =>
|
||||
`${shell.paneKey ?? '-'}=${shell.pid}@${shell.startTicks}`
|
||||
|
||||
const baselinePanes = await readRemotePaneCensus(orcaPage, remote.worktreeId)
|
||||
expect(baselinePanes.paneIds).toHaveLength(PANE_COUNT)
|
||||
const baselineShells = readDockerSshRelayRemotePtys(relayTarget)
|
||||
// Ties every later clause to a real starting point: the whole spec is
|
||||
// vacuous if the panes never owned a shell in the first place.
|
||||
for (const paneKey of paneKeys) {
|
||||
expect(
|
||||
baselineShells.filter((shell) => shell.paneKey === paneKey),
|
||||
`pane ${paneKey} did not start out owning exactly one remote shell`
|
||||
).toHaveLength(1)
|
||||
}
|
||||
const baselineBindings = await distinctBoundPtyIdsByLeaf(orcaPage, hostId, remote.worktreeId)
|
||||
for (const leafId of leafIds) {
|
||||
expect(
|
||||
baselineBindings[leafId] ?? [],
|
||||
`pane ${leafId} must start out durably bound to exactly one shell`
|
||||
).toHaveLength(1)
|
||||
}
|
||||
testInfo.annotations.push({
|
||||
type: 'ssh-partition-cardinality-baseline',
|
||||
description: `${describeDockerSshRelayRemotePtys(baselineShells)} leases=${describeSshRemotePtyLeases(
|
||||
readSshRemotePtyLeases(stateFile, remote.targetId)
|
||||
)}`
|
||||
})
|
||||
|
||||
/** Every dimension a duplicated pane binding moves, re-read from its owner. */
|
||||
const assertCardinalityHolds = async (label: string): Promise<void> => {
|
||||
const panes = await readRemotePaneCensus(orcaPage, remote.worktreeId)
|
||||
expect(panes.paneIds, `${label} changed the panes the user can see`).toEqual(
|
||||
baselinePanes.paneIds
|
||||
)
|
||||
expect(panes.tabIds, `${label} changed the workspace tabs`).toEqual(baselinePanes.tabIds)
|
||||
|
||||
const leases = liveLeasesByLeaf(stateFile, remote.targetId)
|
||||
for (const leafId of leafIds) {
|
||||
expect(
|
||||
leases[leafId] ?? [],
|
||||
`${label} left more than one live lease naming pane ${snapshot.tabId}/${leafId}`
|
||||
).toHaveLength(1)
|
||||
}
|
||||
expect(
|
||||
Object.keys(leases).sort(),
|
||||
`${label} left a live lease for a pane that does not exist`
|
||||
).toEqual([...leafIds].sort())
|
||||
|
||||
const shells = readDockerSshRelayRemotePtys(relayTarget)
|
||||
for (const paneKey of paneKeys) {
|
||||
expect(
|
||||
shells.filter((shell) => shell.paneKey === paneKey),
|
||||
`${label} left the host running more than one shell for pane ${paneKey}`
|
||||
).toHaveLength(1)
|
||||
}
|
||||
expect(shells, `${label} changed the number of shells the host runs`).toHaveLength(
|
||||
PANE_COUNT
|
||||
)
|
||||
|
||||
// The pane may legitimately be bound to a successor shell, but never to
|
||||
// two at once: that is the divergence a user reads as a doubled terminal.
|
||||
const bindings = await distinctBoundPtyIdsByLeaf(orcaPage, hostId, remote.worktreeId)
|
||||
for (const leafId of leafIds) {
|
||||
expect(
|
||||
bindings[leafId] ?? [],
|
||||
`${label} left pane ${leafId} durably bound to more than one shell`
|
||||
).toHaveLength(1)
|
||||
}
|
||||
expect(
|
||||
Object.keys(bindings).sort(),
|
||||
`${label} durably bound a pane the user never opened`
|
||||
).toEqual([...leafIds].sort())
|
||||
}
|
||||
|
||||
for (let cycle = 1; cycle <= RECONNECT_CYCLES; cycle += 1) {
|
||||
await reconnectDockerSshRelayTarget(orcaPage, remote.targetId)
|
||||
await waitForSshConnected(orcaPage, remote.targetId, 180_000)
|
||||
await ensureTerminalVisible(orcaPage, 45_000)
|
||||
await waitForActiveTerminalManager(orcaPage, 60_000)
|
||||
await waitForActivePanePtyId(orcaPage, 60_000)
|
||||
|
||||
// Poll first: a duplicate can land after reattach reports ready, and a
|
||||
// single read taken the instant the wait passes would miss it.
|
||||
await expect
|
||||
.poll(() => readDockerSshRelayRemotePtys(relayTarget).length, {
|
||||
timeout: 120_000,
|
||||
message: `reconnect ${cycle} changed the number of shells the host runs`
|
||||
})
|
||||
.toBe(PANE_COUNT)
|
||||
await assertCardinalityHolds(`reconnect ${cycle}`)
|
||||
|
||||
// Poll returns on its first passing probe, so sit out the settle window
|
||||
// and re-read every dimension a late graft could still move. Attributing
|
||||
// it to the reconnect that caused it beats letting it leak into the next
|
||||
// cycle, where the count is already suspect.
|
||||
await orcaPage.waitForTimeout(SETTLE_MS)
|
||||
await assertCardinalityHolds(`reconnect ${cycle} after settling`)
|
||||
testInfo.annotations.push({
|
||||
type: `ssh-partition-cardinality-reconnect-${cycle}`,
|
||||
description: `${describeDockerSshRelayRemotePtys(
|
||||
readDockerSshRelayRemotePtys(relayTarget)
|
||||
)} leases=${describeSshRemotePtyLeases(readSshRemotePtyLeases(stateFile, remote.targetId))}`
|
||||
})
|
||||
}
|
||||
|
||||
// Closing the loop on the report itself: after the third reconnect the
|
||||
// host runs exactly the shells it ran before the first one.
|
||||
expect(
|
||||
readDockerSshRelayRemotePtys(relayTarget).map(shellIdentity).sort(),
|
||||
`${RECONNECT_CYCLES} reconnects replaced or multiplied the remote shells`
|
||||
).toEqual(baselineShells.map(shellIdentity).sort())
|
||||
} finally {
|
||||
cleanupDockerSshRelayTarget(target)
|
||||
}
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user