mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
Revert "fix(ssh): reapply the reattach-identity work and stop the fallback fence stranding moved panes" (#14395)
This commit is contained in:
Neil
@@ -102,9 +102,6 @@ docs/**
|
||||
!docs/reference/linux-glibc-compatibility.md
|
||||
!docs/reference/relay-grace-time-reconfiguration.md
|
||||
!docs/reference/remote-wire-compatibility.md
|
||||
!docs/reference/terminal-session-behavior-contract.md
|
||||
!docs/reference/terminal-session-correctness-handoff-2026-08-07/
|
||||
!docs/reference/terminal-session-correctness-handoff-2026-08-07/**
|
||||
!docs/reference/renderer-agent-status-performance.md
|
||||
!docs/reference/windows-setup-shell.md
|
||||
!docs/reference/worktree-scan-fingerprint.md
|
||||
|
||||
@@ -6938,7 +6938,7 @@
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/startup/ssh-startup-reconnect.test.ts src/renderer/src/lib/resolved-worktree-execution-host.test.ts src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts src/renderer/src/runtime/web-session-terminal-handle-events.test.ts src/renderer/src/store/slices/terminal-pty-identity-replacement.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/store/slices/direct-ssh-terminal-retry.test.ts src/renderer/src/store/slices/direct-ssh-pane-detach-ledger.test.ts src/renderer/src/store/slices/direct-ssh-terminal-recovery.test.ts src/renderer/src/store/slices/direct-ssh-terminal-workspace-scope.test.ts src/renderer/src/store/slices/terminals-hydration.test.ts src/renderer/src/store/slices/repos-ssh-host-reconciliation.test.ts src/renderer/src/hooks/direct-ssh-reconnect-coordinator.test.ts src/renderer/src/hooks/direct-ssh-host-hydration.test.ts src/renderer/src/hooks/direct-ssh-state-routing.test.ts src/renderer/src/hooks/remote-workspace-target-sync.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/repos-remote.test.ts src/main/ipc/ssh.test.ts src/main/ipc/worktrees.test.ts src/main/runtime/public-ssh-state.test.ts src/main/ssh/ssh-connection-manager.test.ts src/main/ssh/ssh-connection.test.ts src/main/ssh/ssh-provider-authority.test.ts src/preload/ssh-authority-forwarding.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/runtime/runtime-environment-ssh-state.test.ts src/shared/ssh-retained-payload-admission.test.ts src/shared/ssh-types.test.ts --reporter=dot",
|
||||
"pnpm exec electron-vite build --mode e2e",
|
||||
@@ -6959,7 +6959,7 @@
|
||||
"src/renderer/src/components/terminal-pane/pty-transport.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx",
|
||||
"src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx",
|
||||
"src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts",
|
||||
"src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts",
|
||||
"src/renderer/src/runtime/web-session-terminal-handle-events.test.ts",
|
||||
@@ -7041,7 +7041,7 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx",
|
||||
"file": "src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx",
|
||||
"assertions": [
|
||||
"automatic recovery renders as a quiet bounded status without a button",
|
||||
"the disconnected state exposes one explicit Reconnect action"
|
||||
@@ -7316,7 +7316,7 @@
|
||||
"date": "2026-07-21",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 5,
|
||||
"summary": "Eleven fault-injection and recovery-UI files and 127 tests passed, covering suspended heartbeat clocks, stale socket, PTY, and create generations, canonical pre-ready close recovery with one replacement subscription, cold and repeated PTY reattachment, authoritative health, bounded PTY and terminal-create recovery, post-probe timeout clipping, manually retryable create cutoff, accurate capability-probe failures, same-PTY manual reconnect, pane-state cleanup, fatal error deduplication, stable create identity, cross-process PTY adoption, and fail-closed legacy inventory."
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Static source assertions for the launcher's hold decision.
|
||||
*
|
||||
* Split from the repro script it serves: those phases prove behaviour with real processes,
|
||||
* while these read `daemon-init.ts` to pin the one property real processes cannot reach —
|
||||
* that the decision is taken, and returns, before anything is killed. daemon-init.ts imports
|
||||
* electron, so it cannot be executed outside the app.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { join, resolve } from 'node:path'
|
||||
|
||||
const repoRoot = resolve(import.meta.dirname, '..', '..')
|
||||
|
||||
export function stripComments(source) {
|
||||
// Blanked rather than deleted so offsets and line numbers stay true to the real file.
|
||||
const blank = (text) => text.replace(/[^\n]/g, ' ')
|
||||
return source
|
||||
.replace(/\/\*[\s\S]*?\*\//g, blank)
|
||||
.replace(/(^|[^:])(\/\/[^\n]*)/g, (_match, prefix, comment) => prefix + blank(comment))
|
||||
}
|
||||
|
||||
/** The balanced `{...}` block starting at `braceIndex`, or null if it never closes. */
|
||||
function extractBlock(source, braceIndex) {
|
||||
let depth = 0
|
||||
for (let i = braceIndex; i < source.length; i++) {
|
||||
if (source[i] === '{') {
|
||||
depth++
|
||||
} else if (source[i] === '}') {
|
||||
depth--
|
||||
if (depth === 0) {
|
||||
return { text: source.slice(braceIndex, i + 1), start: braceIndex, end: i + 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function lineOf(source, index) {
|
||||
return source.slice(0, index).split('\n').length
|
||||
}
|
||||
|
||||
function normalize(text) {
|
||||
return text.replace(/\s+/g, ' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* PHASE 3 — the launcher must hold rather than kill. daemon-init.ts imports electron, so it
|
||||
* cannot be executed here; this reads the source instead, whitespace-tolerantly, and asserts
|
||||
* the structural properties phase 2's inputs depend on.
|
||||
*/
|
||||
export function checkLauncherHoldsOccupiedDaemon({ log, assert }) {
|
||||
const relativePath = 'src/main/daemon/daemon-init.ts'
|
||||
const source = stripComments(readFileSync(join(repoRoot, relativePath), 'utf8'))
|
||||
|
||||
// 1. holdIncumbentDaemon() returns a preserved handle in 'held' mode — it does not adopt,
|
||||
// which a daemon too wedged to answer listSessions could never complete anyway.
|
||||
const holdDecl = source.match(/const\s+holdIncumbentDaemon\s*=\s*\([^)]*\)[^{]*\{/)
|
||||
assert(holdDecl !== null, `${relativePath} does not declare holdIncumbentDaemon()`)
|
||||
const holdBody = extractBlock(source, holdDecl.index + holdDecl[0].length - 1)
|
||||
assert(holdBody !== null, `could not parse the holdIncumbentDaemon() body in ${relativePath}`)
|
||||
assert(
|
||||
/createPreservedDaemonHandle\([^)]*'held'\s*\)/.test(normalize(holdBody.text)),
|
||||
`holdIncumbentDaemon() does not return createPreservedDaemonHandle(..., 'held'): ${normalize(holdBody.text)}`
|
||||
)
|
||||
log(
|
||||
`phase 3: ${relativePath}:${lineOf(source, holdDecl.index)} holdIncumbentDaemon() = ${normalize(holdBody.text)}`
|
||||
)
|
||||
|
||||
// 2. Process-table evidence is only ever raised from an identity-verified pid — otherwise
|
||||
// it could describe a recycled pid's children rather than this daemon's terminals.
|
||||
const verifiedPidCall = source.search(/readVerifiedDaemonPid\s*\(/)
|
||||
const evidenceCall = source.match(/raiseOccupancyWithProcessEvidence\s*\(([^)]*)\)/)
|
||||
assert(verifiedPidCall !== -1, `${relativePath} never calls readVerifiedDaemonPid()`)
|
||||
assert(evidenceCall !== null, `${relativePath} never raises occupancy with process evidence`)
|
||||
assert(
|
||||
verifiedPidCall < evidenceCall.index,
|
||||
`${relativePath} raises occupancy with process evidence before verifying the recorded pid`
|
||||
)
|
||||
// Whatever identifier carries the pid, its declaration must come from the verified read.
|
||||
const evidencePidName = evidenceCall[1]
|
||||
.split(',')[1]
|
||||
?.trim()
|
||||
.replace(/[^\w$]/g, '')
|
||||
assert(
|
||||
Boolean(evidencePidName),
|
||||
`could not read the pid argument of raiseOccupancyWithProcessEvidence: ${normalize(evidenceCall[1])}`
|
||||
)
|
||||
const evidencePidDecl = new RegExp(
|
||||
`const\\s+${evidencePidName}\\b[\\s\\S]{0,400}?readVerifiedDaemonPid\\s*\\(`
|
||||
)
|
||||
assert(
|
||||
evidencePidDecl.test(source),
|
||||
`${relativePath} passes '${evidencePidName}' to raiseOccupancyWithProcessEvidence without deriving it from readVerifiedDaemonPid — the evidence could then describe a recycled pid's children`
|
||||
)
|
||||
|
||||
// 3. The 'occupied' branch holds and never kills.
|
||||
const occupiedGuard = source.match(/if\s*\(\s*occupancy\.state\s*===\s*'occupied'\s*\)\s*\{/)
|
||||
assert(occupiedGuard !== null, `${relativePath} has no 'occupancy.state === occupied' guard`)
|
||||
const occupiedBlock = extractBlock(source, occupiedGuard.index + occupiedGuard[0].length - 1)
|
||||
assert(occupiedBlock !== null, `could not parse the occupied branch in ${relativePath}`)
|
||||
const occupiedLine = lineOf(source, occupiedGuard.index)
|
||||
assert(
|
||||
!occupiedBlock.text.includes('killStaleDaemon'),
|
||||
`${relativePath}:${occupiedLine} calls killStaleDaemon inside the occupied branch`
|
||||
)
|
||||
|
||||
// Holding requires BOTH: no hello ever completed, and only the process table could answer.
|
||||
// A daemon that did complete a hello is adoptable, so it must not be routed to a mode that
|
||||
// never adopts.
|
||||
const unverifiableGuard = occupiedBlock.text.match(
|
||||
/if\s*\(\s*health\s*===\s*'rejected'\s*\|\|\s*occupancy\.liveSessions\s*===\s*null\s*\)\s*\{/
|
||||
)
|
||||
assert(
|
||||
unverifiableGuard !== null,
|
||||
`${relativePath}:${occupiedLine} does not gate the hold on a daemon that cannot be adopted (rejected, or an unverifiable session count)`
|
||||
)
|
||||
const unverifiableBlock = extractBlock(
|
||||
occupiedBlock.text,
|
||||
unverifiableGuard.index + unverifiableGuard[0].length - 1
|
||||
)
|
||||
assert(unverifiableBlock !== null, 'could not parse the liveSessions === null branch')
|
||||
assert(
|
||||
normalize(unverifiableBlock.text).includes('return holdIncumbentDaemon()'),
|
||||
`${relativePath}:${occupiedLine} does not return holdIncumbentDaemon() when the session count came from the process table`
|
||||
)
|
||||
log(
|
||||
`phase 3: ${relativePath}:${occupiedLine} occupancy.state === 'occupied' + cannot-be-adopted -> return holdIncumbentDaemon(); the branch contains no kill`
|
||||
)
|
||||
|
||||
// 3b. The unknown-hold: the protection that no longer depends on any timing budget. An
|
||||
// unclassifiable daemon is held, not replaced, except where holding is unrecoverable.
|
||||
const unknownHold = source.match(
|
||||
/if\s*\(\s*occupancy\.state === 'unknown' &&[\s\S]{0,1500}?return holdIncumbentDaemon\(\)/
|
||||
)
|
||||
assert(
|
||||
unknownHold !== null,
|
||||
`${relativePath} does not hold on occupancy.state === 'unknown' — a daemon we could not classify is being replaced`
|
||||
)
|
||||
assert(
|
||||
unknownHold[0].includes("health !== 'rejected'"),
|
||||
`the unknown-hold does not exclude 'rejected', which can never be adopted: ${normalize(unknownHold[0])}`
|
||||
)
|
||||
assert(
|
||||
unknownHold[0].includes('endpointIsProvenDead'),
|
||||
`the unknown-hold does not exclude a proven-dead endpoint, so a cold start would be held: ${normalize(unknownHold[0])}`
|
||||
)
|
||||
log(
|
||||
`phase 3: ${relativePath}:${lineOf(source, unknownHold.index)} occupancy.state === 'unknown' + not-proven-dead + not-rejected -> return holdIncumbentDaemon()`
|
||||
)
|
||||
|
||||
// 4. Ordering: every kill on this path is downstream of the occupied branch, so a hold
|
||||
// returns before any of them can run.
|
||||
const killCalls = [...source.matchAll(/killStaleDaemon\s*\(/g)].map((match) => match.index)
|
||||
assert(killCalls.length > 0, `${relativePath} never calls killStaleDaemon()`)
|
||||
const killsBeforeTheDecision = killCalls.filter(
|
||||
(index) => index > evidenceCall.index && index < occupiedBlock.end
|
||||
)
|
||||
assert(
|
||||
killsBeforeTheDecision.length === 0,
|
||||
`${relativePath} kills at line(s) ${killsBeforeTheDecision.map((i) => lineOf(source, i)).join(', ')}, between resolving occupancy and the hold`
|
||||
)
|
||||
const killsBeforeTheUnknownHold = killCalls.filter(
|
||||
(index) => index > evidenceCall.index && index < unknownHold.index
|
||||
)
|
||||
assert(
|
||||
killsBeforeTheUnknownHold.length === 0,
|
||||
`${relativePath} kills at line(s) ${killsBeforeTheUnknownHold.map((i) => lineOf(source, i)).join(', ')}, before the unknown-hold can return`
|
||||
)
|
||||
const fallThroughKill = killCalls.find((index) => index > unknownHold.index)
|
||||
assert(
|
||||
fallThroughKill !== undefined,
|
||||
`${relativePath} has no killStaleDaemon() after the occupied branch — the replacement path is gone`
|
||||
)
|
||||
const killLines = killCalls.map((index) => lineOf(source, index)).join(', ')
|
||||
log(
|
||||
`phase 3: every killStaleDaemon() call site in the file is at line(s) ${killLines} — all downstream of the occupied branch, which returns at line ${lineOf(source, occupiedBlock.start)}`
|
||||
)
|
||||
log(
|
||||
'phase 3 RESULT: statically, the failed-health-check path resolves occupancy from a verified pid and returns a held handle before any kill. This proves the source ordering and branch contents; it does NOT execute daemon-init.ts (it imports electron), so the runtime proof stops at the inputs phase 2 produced with real processes.'
|
||||
)
|
||||
}
|
||||
@@ -1,553 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Regression proof: daemon replacement must not kill live coding-agent terminals.
|
||||
*
|
||||
* The protection is no longer a veto inside `killStaleDaemon()` — that was policy
|
||||
* buried in a mechanism. `killStaleDaemon()` is now purely "make this pid go away"
|
||||
* and will happily kill a daemon that is hosting live agents. The decision moved
|
||||
* up to the launcher, ahead of any kill:
|
||||
*
|
||||
* readVerifiedDaemonPid() -> which process, identity-verified, is the daemon
|
||||
* resolveDaemonOccupancy() -> is it hosting work, and how sure are we
|
||||
* daemon-init.ts -> 'occupied' with an unverifiable count => HOLD
|
||||
*
|
||||
* `resolveDaemonOccupancy()` asks the daemon over IPC first (a reply is
|
||||
* authoritative both ways); only when it cannot answer does it consult the OS
|
||||
* process table via `inspectDaemonPtyOwnership()`, and that evidence may only
|
||||
* RAISE the answer to 'occupied' — it can never prove 'empty'.
|
||||
*
|
||||
* Three phases, real processes throughout:
|
||||
* PHASE 1 (the danger is real): a SIGSTOPped daemon owning 2 live agent
|
||||
* processes presents exactly the launcher's inputs — health 'unreachable',
|
||||
* an endpoint that is NOT proven dead, no IPC session count. Calling
|
||||
* killStaleDaemon() directly at that moment kills the daemon and both agents.
|
||||
* This is what the decision is protecting against, not a bug in the kill.
|
||||
* PHASE 2 (the decision protects it): same staging, fresh daemon and agents.
|
||||
* readVerifiedDaemonPid() names the daemon, resolveDaemonOccupancy() returns
|
||||
* { state: 'occupied', liveSessions: null } — IPC could not answer, the
|
||||
* process table raised it to occupied — which is the exact input that makes
|
||||
* the launcher hold. Nothing is signalled: daemon and agents are alive, and
|
||||
* after SIGCONT the daemon is healthy and reports its 2 sessions again, so
|
||||
* the wedge was transient and the preserved work was genuinely recoverable.
|
||||
* PHASE 3 (the launcher actually holds): daemon-init.ts imports electron and
|
||||
* cannot be executed here, so its failed-health-check branch is verified
|
||||
* statically — the 'occupied' branch returns holdIncumbentDaemon() and
|
||||
* contains no kill, and every killStaleDaemon() call sits after it.
|
||||
*
|
||||
* SIGSTOP is the faithful stand-in for the wedge: the socket still accepts
|
||||
* connections while no RPC is ever answered — exactly the "busy machine can time
|
||||
* out the health check on a live daemon" case daemon-init.ts calls out.
|
||||
*
|
||||
* Usage: node config/scripts/daemon-replacement-live-agent-pty-preservation-repro.mjs
|
||||
*/
|
||||
import { fork } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { checkLauncherHoldsOccupiedDaemon } from './daemon-replacement-launcher-hold-source-assertions.mjs'
|
||||
import {
|
||||
findTaggedPid,
|
||||
isMarkerAlive,
|
||||
verifiedSessionLeaderPid,
|
||||
isProcessAlive,
|
||||
processArgs,
|
||||
processState,
|
||||
snapshotForeignDaemons,
|
||||
waitFor
|
||||
} from './daemon-replacement-process-inspection.mjs'
|
||||
|
||||
const repoRoot = resolve(import.meta.dirname, '..', '..')
|
||||
const entryPath = join(repoRoot, 'out', 'main', 'daemon-entry.js')
|
||||
const READY_TIMEOUT_MS = 30_000
|
||||
const MARKER_SPAWN_TIMEOUT_MS = 30_000
|
||||
const SESSION_COUNT = 2
|
||||
|
||||
const startedAt = Date.now()
|
||||
const timeline = []
|
||||
|
||||
function log(message) {
|
||||
const elapsed = `+${String(Date.now() - startedAt).padStart(6, ' ')}ms`
|
||||
timeline.push(`${elapsed} ${message}`)
|
||||
process.stdout.write(`[daemon-pty-preservation] ${elapsed} ${message}\n`)
|
||||
}
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) {
|
||||
throw new Error(message)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bundles the real daemon primitives into a loadable ESM module.
|
||||
*
|
||||
* Why: the decision primitives live in TypeScript modules that the built
|
||||
* daemon-entry.js does not re-export. Their import graph is electron-free, so
|
||||
* esbuild can produce the genuine code — no reimplementation, no drift.
|
||||
*/
|
||||
async function loadDaemonPrimitives(scratch) {
|
||||
const esbuild = await import('esbuild')
|
||||
const entrySource = join(scratch, 'daemon-primitives-entry.ts')
|
||||
const bundlePath = join(scratch, 'daemon-primitives.mjs')
|
||||
const daemonDir = join(repoRoot, 'src', 'main', 'daemon')
|
||||
writeFileSync(
|
||||
entrySource,
|
||||
[
|
||||
`export { checkDaemonHealth, killStaleDaemon, readVerifiedDaemonPid } from ${JSON.stringify(join(daemonDir, 'daemon-health'))}`,
|
||||
`export { resolveDaemonOccupancy } from ${JSON.stringify(join(daemonDir, 'daemon-occupancy'))}`,
|
||||
`export { endpointIsProvenDead, probeSocketConnect } from ${JSON.stringify(join(daemonDir, 'daemon-endpoint-probe'))}`,
|
||||
`export { getDaemonPidPath, getDaemonSocketPath, getDaemonTokenPath } from ${JSON.stringify(join(daemonDir, 'daemon-spawner'))}`,
|
||||
`export { DaemonClient } from ${JSON.stringify(join(daemonDir, 'client'))}`,
|
||||
''
|
||||
].join('\n')
|
||||
)
|
||||
await esbuild.build({
|
||||
entryPoints: [entrySource],
|
||||
outfile: bundlePath,
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
packages: 'external',
|
||||
logLevel: 'silent'
|
||||
})
|
||||
return import(pathToFileURL(bundlePath).href)
|
||||
}
|
||||
|
||||
// Same shape as daemon-occupancy.ts countLiveSessionsOverIpc(): null means "could not answer".
|
||||
async function countLiveSessionsOverIpc(DaemonClient, socketPath, tokenPath) {
|
||||
const client = new DaemonClient({ socketPath, tokenPath })
|
||||
try {
|
||||
await client.ensureConnected()
|
||||
const result = await client.request('listSessions', undefined)
|
||||
return result.sessions.filter((session) => session.isAlive).length
|
||||
} catch {
|
||||
return null
|
||||
} finally {
|
||||
client.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
function forkDaemon({ runtimeDir, socketPath, tokenPath, pidPath, launchNonce, logFile }) {
|
||||
// Argv and spawn options mirror daemon-init.ts createOutOfProcessLauncher().
|
||||
const child = fork(
|
||||
entryPath,
|
||||
[
|
||||
'--socket',
|
||||
socketPath,
|
||||
'--token',
|
||||
tokenPath,
|
||||
'--pid-record',
|
||||
pidPath,
|
||||
'--launch-nonce',
|
||||
launchNonce,
|
||||
'--entry-path',
|
||||
entryPath,
|
||||
'--app-version',
|
||||
'daemon-pty-preservation-repro',
|
||||
'--spawner-exec-path',
|
||||
process.execPath,
|
||||
'--log-file',
|
||||
logFile
|
||||
],
|
||||
{
|
||||
cwd: runtimeDir,
|
||||
detached: true,
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
ORCA_USER_DATA_PATH: runtimeDir
|
||||
}
|
||||
}
|
||||
)
|
||||
let stderr = ''
|
||||
child.stderr?.on('data', (chunk) => {
|
||||
stderr += chunk.toString('utf8')
|
||||
})
|
||||
const ready = new Promise((resolveReady, rejectReady) => {
|
||||
const timer = setTimeout(
|
||||
() => rejectReady(new Error(`daemon never signaled ready.\nstderr:\n${stderr}`)),
|
||||
READY_TIMEOUT_MS
|
||||
)
|
||||
child.on('message', (msg) => {
|
||||
if (msg && typeof msg === 'object' && msg.type === 'ready') {
|
||||
clearTimeout(timer)
|
||||
resolveReady()
|
||||
}
|
||||
})
|
||||
child.on('exit', (code, signal) => {
|
||||
clearTimeout(timer)
|
||||
rejectReady(new Error(`daemon exited (code=${code}, signal=${signal}).\nstderr:\n${stderr}`))
|
||||
})
|
||||
})
|
||||
return { child, ready }
|
||||
}
|
||||
|
||||
async function startMarkerSession(client, phase, index, runtimeDir) {
|
||||
const tag = `ORCA_LIVE_AGENT_MARKER_P${phase}_${index}_${randomUUID().replaceAll('-', '')}`
|
||||
const sessionId = `repro-session-${phase}-${index}-${randomUUID()}`
|
||||
// Long-lived and uniquely identifiable: stands in for a running coding agent.
|
||||
const command = `exec /bin/sh -c 'while :; do sleep 1; done' ${tag}`
|
||||
const result = await client.request('createOrAttach', {
|
||||
sessionId,
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: runtimeDir,
|
||||
command,
|
||||
shellReadySupported: false
|
||||
})
|
||||
if (!Number.isInteger(result.pid) || result.pid <= 0) {
|
||||
throw new Error(`session ${index} reported no pid: ${JSON.stringify(result)}`)
|
||||
}
|
||||
let markerPid = null
|
||||
await waitFor(
|
||||
() => (markerPid = findTaggedPid(tag)) !== null,
|
||||
`agent marker ${index} to start`,
|
||||
MARKER_SPAWN_TIMEOUT_MS
|
||||
)
|
||||
return { tag, sessionId, pid: markerPid, sessionPid: result.pid }
|
||||
}
|
||||
|
||||
/**
|
||||
* Stands up a real daemon with real agent processes, wedges it with SIGSTOP, and replays
|
||||
* the launcher's decision inputs against it — the state both phases start from.
|
||||
*/
|
||||
async function stageWedgedDaemon({ primitives, scratch, phase, registry }) {
|
||||
const { DaemonClient, checkDaemonHealth, endpointIsProvenDead, probeSocketConnect } = primitives
|
||||
const runtimeDir = join(scratch, `daemon-phase-${phase}`)
|
||||
mkdirSync(runtimeDir, { recursive: true })
|
||||
const socketPath = primitives.getDaemonSocketPath(runtimeDir)
|
||||
const tokenPath = primitives.getDaemonTokenPath(runtimeDir)
|
||||
const pidPath = primitives.getDaemonPidPath(runtimeDir)
|
||||
log(`phase ${phase}: runtime dir ${runtimeDir} (real userData is untouched)`)
|
||||
|
||||
const daemon = forkDaemon({
|
||||
runtimeDir,
|
||||
socketPath,
|
||||
tokenPath,
|
||||
pidPath,
|
||||
launchNonce: randomUUID(),
|
||||
logFile: join(scratch, `daemon-phase-${phase}.log`)
|
||||
})
|
||||
const staged = { daemon, markers: [], stopped: false, runtimeDir, socketPath, tokenPath, pidPath }
|
||||
// Registered before the first await so a mid-staging failure still tears it down.
|
||||
registry.push(staged)
|
||||
await daemon.ready
|
||||
log(`phase ${phase}: daemon ready, pid ${daemon.child.pid}`)
|
||||
|
||||
const client = new DaemonClient({ socketPath, tokenPath })
|
||||
await client.ensureConnected()
|
||||
for (let index = 0; index < SESSION_COUNT; index++) {
|
||||
staged.markers.push(await startMarkerSession(client, phase, index, runtimeDir))
|
||||
}
|
||||
const liveBefore = await countLiveSessionsOverIpc(DaemonClient, socketPath, tokenPath)
|
||||
client.disconnect()
|
||||
for (const marker of staged.markers) {
|
||||
log(
|
||||
`phase ${phase}: live agent process pid ${marker.pid} (PTY session leader ${marker.sessionPid}): ${processArgs(marker.pid)}`
|
||||
)
|
||||
}
|
||||
assert(staged.markers.every(isMarkerAlive), 'agent markers were not alive before the wedge')
|
||||
log(
|
||||
`phase ${phase}: ps confirms ${staged.markers.length} live agent processes; daemon reports ${liveBefore} alive`
|
||||
)
|
||||
|
||||
process.kill(daemon.child.pid, 'SIGSTOP')
|
||||
staged.stopped = true
|
||||
log(`phase ${phase}: SIGSTOP -> daemon ${daemon.child.pid} is ALIVE but cannot service RPCs`)
|
||||
assert(staged.markers.every(isMarkerAlive), 'the wedge itself killed the agent markers')
|
||||
log(`phase ${phase}: agent processes unaffected by the wedge — only the daemon is unresponsive`)
|
||||
|
||||
// The launcher's own inputs on the failed-health-check path, via the real primitives.
|
||||
const health = await checkDaemonHealth(socketPath, tokenPath)
|
||||
log(`phase ${phase}: checkDaemonHealth() = '${health}' — daemon-init.ts takes the else branch`)
|
||||
assert(health === 'unreachable', `expected health 'unreachable', got '${health}'`)
|
||||
|
||||
const probe = await probeSocketConnect(socketPath)
|
||||
log(
|
||||
`phase ${phase}: probeSocketConnect() = '${probe}', endpointIsProvenDead() = ${endpointIsProvenDead(probe)} — nothing proves the daemon is gone`
|
||||
)
|
||||
assert(
|
||||
!endpointIsProvenDead(probe),
|
||||
`the wedged daemon's endpoint was proven dead ('${probe}'); not the modeled failure`
|
||||
)
|
||||
|
||||
const ipcCount = await countLiveSessionsOverIpc(DaemonClient, socketPath, tokenPath)
|
||||
log(
|
||||
`phase ${phase}: live session count over IPC = ${ipcCount} (null = the daemon could not answer)`
|
||||
)
|
||||
assert(ipcCount === null, 'the wedged daemon answered listSessions; wedge not severe enough')
|
||||
|
||||
return staged
|
||||
}
|
||||
|
||||
/**
|
||||
* PHASE 1 — what the decision is protecting against. killStaleDaemon() is now a pure
|
||||
* mechanism with no opinion about live work, so called at this exact moment it takes the
|
||||
* daemon and every agent PTY with it.
|
||||
*/
|
||||
async function runUnprotectedKillPhase(primitives, scratch, registry) {
|
||||
const staged = await stageWedgedDaemon({ primitives, scratch, phase: 1, registry })
|
||||
log(
|
||||
'phase 1: invoking the real killStaleDaemon(runtimeDir, socket, token) directly — no occupancy consulted'
|
||||
)
|
||||
const killOutcome = await primitives.killStaleDaemon(
|
||||
staged.runtimeDir,
|
||||
staged.socketPath,
|
||||
staged.tokenPath
|
||||
)
|
||||
log(`phase 1: killStaleDaemon() = ${JSON.stringify(killOutcome)}`)
|
||||
staged.stopped = false
|
||||
assert(killOutcome.killed === true, 'killStaleDaemon() did not kill the wedged daemon')
|
||||
assert(!isProcessAlive(staged.daemon.child.pid), 'killStaleDaemon() left the daemon alive')
|
||||
|
||||
await waitFor(
|
||||
() => staged.markers.every((marker) => !isMarkerAlive(marker)),
|
||||
'agent processes to die with the killed daemon',
|
||||
10_000
|
||||
)
|
||||
for (const marker of staged.markers) {
|
||||
log(
|
||||
`phase 1: agent PTY pid ${marker.pid} is GONE (ps: ${processArgs(marker.pid) ?? 'no such process'})`
|
||||
)
|
||||
}
|
||||
log(
|
||||
'phase 1 RESULT: the danger is real — killStaleDaemon() on a wedged-but-live daemon ends the daemon and every agent with it. There is no fd handoff; only a decision taken BEFORE the kill can save them.'
|
||||
)
|
||||
return staged
|
||||
}
|
||||
|
||||
/**
|
||||
* PHASE 2 — the decision the launcher takes instead. Identical staging, but the inputs are
|
||||
* resolved rather than acted on: readVerifiedDaemonPid() names the process and
|
||||
* resolveDaemonOccupancy() raises it to 'occupied' from the process table.
|
||||
*/
|
||||
async function runOccupancyDecisionPhase(primitives, scratch, registry) {
|
||||
const staged = await stageWedgedDaemon({ primitives, scratch, phase: 2, registry })
|
||||
const daemonPid = staged.daemon.child.pid
|
||||
|
||||
const verifiedPid = await primitives.readVerifiedDaemonPid(
|
||||
staged.runtimeDir,
|
||||
staged.socketPath,
|
||||
staged.tokenPath
|
||||
)
|
||||
log(
|
||||
`phase 2: readVerifiedDaemonPid() = ${verifiedPid ? `pid ${verifiedPid.pid} (identity verified: cmdline + start time)` : 'null'}`
|
||||
)
|
||||
assert(
|
||||
verifiedPid?.pid === daemonPid,
|
||||
`readVerifiedDaemonPid() returned ${JSON.stringify(verifiedPid)}, expected pid ${daemonPid}`
|
||||
)
|
||||
|
||||
// Which input answered is readable from the result alone: resolveDaemonOccupancy only ever
|
||||
// returns a null count when IPC failed and inspectDaemonPtyOwnership() — the OS process
|
||||
// table, never the socket the daemon already failed to answer — reported 'owns-live-ptys'.
|
||||
let occupancy = await primitives.resolveDaemonOccupancy({
|
||||
socketPath: staged.socketPath,
|
||||
tokenPath: staged.tokenPath,
|
||||
recordedPid: verifiedPid.pid
|
||||
})
|
||||
log(`phase 2: resolveDaemonOccupancy() = ${JSON.stringify(occupancy)}`)
|
||||
// The launcher's grace loop, replayed verbatim: it only re-samples while 'unknown'.
|
||||
let graceRetry = 0
|
||||
while (
|
||||
occupancy.state === 'unknown' &&
|
||||
graceRetry < 1 &&
|
||||
!primitives.endpointIsProvenDead(await primitives.probeSocketConnect(staged.socketPath))
|
||||
) {
|
||||
occupancy = await primitives.resolveDaemonOccupancy({
|
||||
socketPath: staged.socketPath,
|
||||
tokenPath: staged.tokenPath,
|
||||
recordedPid: verifiedPid.pid
|
||||
})
|
||||
graceRetry++
|
||||
}
|
||||
log(
|
||||
`phase 2: the launcher makes one patient ask and no retries — what remains after the patient connect is always exactly the request budget, which cannot fund another (ran ${graceRetry})`
|
||||
)
|
||||
assert(
|
||||
occupancy.state === 'occupied' && occupancy.liveSessions === null,
|
||||
`expected {state:'occupied',liveSessions:null}, got ${JSON.stringify(occupancy)}`
|
||||
)
|
||||
log(
|
||||
"phase 2: occupancy is 'occupied' with liveSessions null — IPC could not answer, so the count came from the process table. That exact pair is what makes the launcher hold instead of kill (phase 3)."
|
||||
)
|
||||
|
||||
assert(isProcessAlive(daemonPid), 'the daemon died while occupancy was being resolved')
|
||||
log(
|
||||
`phase 2: daemon ${daemonPid} is STILL ALIVE (ps stat '${processState(daemonPid)}' — T = stopped, not killed); resolving occupancy signals nothing`
|
||||
)
|
||||
assert(existsSync(staged.pidPath), 'the surviving daemon lost its PID record')
|
||||
log('phase 2: PID record left intact — no replacement can publish ownership beside it')
|
||||
|
||||
for (const marker of staged.markers) {
|
||||
assert(isMarkerAlive(marker), `agent PTY pid ${marker.pid} died during the decision`)
|
||||
log(`phase 2: agent PTY pid ${marker.pid} is ALIVE (ps: ${processArgs(marker.pid)})`)
|
||||
}
|
||||
|
||||
// Why SIGCONT: a SIGTERM sent to a stopped process stays pending and lands on
|
||||
// resume. Surviving the resume is the proof that no signal was even queued.
|
||||
process.kill(daemonPid, 'SIGCONT')
|
||||
staged.stopped = false
|
||||
await new Promise((r) => setTimeout(r, 1_000))
|
||||
assert(isProcessAlive(daemonPid), 'the daemon died on SIGCONT — a SIGTERM had been queued for it')
|
||||
log('phase 2: after SIGCONT the daemon is still running — no signal was ever delivered to it')
|
||||
|
||||
const resumedHealth = await primitives.checkDaemonHealth(staged.socketPath, staged.tokenPath)
|
||||
const resumedSessions = await countLiveSessionsOverIpc(
|
||||
primitives.DaemonClient,
|
||||
staged.socketPath,
|
||||
staged.tokenPath
|
||||
)
|
||||
log(
|
||||
`phase 2: resumed daemon reports checkDaemonHealth() = '${resumedHealth}', live sessions over IPC = ${resumedSessions}`
|
||||
)
|
||||
assert(resumedHealth === 'healthy', `resumed daemon is not healthy: '${resumedHealth}'`)
|
||||
assert(resumedSessions === SESSION_COUNT, `resumed daemon lost sessions: ${resumedSessions}`)
|
||||
for (const marker of staged.markers) {
|
||||
assert(isMarkerAlive(marker), `agent PTY pid ${marker.pid} died during resume`)
|
||||
}
|
||||
const resumedOccupancy = await primitives.resolveDaemonOccupancy({
|
||||
socketPath: staged.socketPath,
|
||||
tokenPath: staged.tokenPath,
|
||||
recordedPid: verifiedPid.pid
|
||||
})
|
||||
log(
|
||||
`phase 2: resolveDaemonOccupancy() on the recovered daemon = ${JSON.stringify(resumedOccupancy)} — the count is authoritative again now that IPC answers`
|
||||
)
|
||||
assert(
|
||||
resumedOccupancy.state === 'occupied' && resumedOccupancy.liveSessions === SESSION_COUNT,
|
||||
`expected {state:'occupied',liveSessions:${SESSION_COUNT}} after recovery, got ${JSON.stringify(resumedOccupancy)}`
|
||||
)
|
||||
log(
|
||||
'phase 2 RESULT: the wedge was transient and the work was genuinely recoverable — the daemon and both agents survived, then came back healthy with all sessions intact'
|
||||
)
|
||||
return staged
|
||||
}
|
||||
|
||||
function teardown(staged) {
|
||||
if (!staged) {
|
||||
return
|
||||
}
|
||||
// Why the exit check: phase 1 kills this daemon on purpose, and once Node has reaped the
|
||||
// child its pid is free for the OS to reuse. Signalling the remembered number after that is
|
||||
// signalling a stranger.
|
||||
const daemonChild = staged.daemon?.child
|
||||
const daemonPid =
|
||||
daemonChild && daemonChild.exitCode === null && daemonChild.signalCode === null
|
||||
? daemonChild.pid
|
||||
: undefined
|
||||
if (daemonPid) {
|
||||
for (const signal of staged.stopped ? ['SIGCONT', 'SIGKILL'] : ['SIGKILL']) {
|
||||
try {
|
||||
process.kill(daemonPid, signal)
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
}
|
||||
staged.daemon.child.stderr?.destroy()
|
||||
if (staged.daemon.child.connected) {
|
||||
staged.daemon.child.disconnect()
|
||||
}
|
||||
staged.daemon.child.unref()
|
||||
}
|
||||
for (const marker of staged.markers ?? []) {
|
||||
// Why re-verify by tag: phase 1 waits for these pids to die, and teardown runs a minute
|
||||
// later. Signalling a remembered pid after that would be signalling whatever the OS has
|
||||
// since recycled it onto — which is the mistake this whole script exists to study.
|
||||
if (!isMarkerAlive(marker)) {
|
||||
continue
|
||||
}
|
||||
// The leader is re-read from the live marker rather than remembered: the marker proves its
|
||||
// own identity by tag, but nothing proved the leader's, and it is the one pid here that
|
||||
// could have been recycled while its child stayed alive under a new parent.
|
||||
for (const pid of [marker.pid, verifiedSessionLeaderPid(marker)]) {
|
||||
if (!pid) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
process.kill(pid, 'SIGKILL')
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.platform === 'win32') {
|
||||
log('SKIP: SIGSTOP is POSIX-only, so a live-but-unresponsive daemon cannot be staged here')
|
||||
return
|
||||
}
|
||||
if (!existsSync(entryPath)) {
|
||||
throw new Error(`missing ${entryPath} — run \`pnpm run build:electron-vite\` first`)
|
||||
}
|
||||
|
||||
const scratch = mkdtempSync(join(tmpdir(), 'orca-dpp-'))
|
||||
const foreignDaemons = snapshotForeignDaemons()
|
||||
const staged = []
|
||||
let verdict = 'FAIL'
|
||||
|
||||
try {
|
||||
log(
|
||||
`pre-existing daemons that must survive this run: ${foreignDaemons
|
||||
.map((d) => `${d.pid}${d.isRealUserDaemon ? ' (real userData daemon)' : ''}`)
|
||||
.join(', ')}`
|
||||
)
|
||||
const primitives = await loadDaemonPrimitives(scratch)
|
||||
|
||||
log('=== PHASE 1: the danger is real — killStaleDaemon() has no opinion about live work ===')
|
||||
await runUnprotectedKillPhase(primitives, scratch, staged)
|
||||
|
||||
log('=== PHASE 2: the decision protects it — resolveDaemonOccupancy() on the same wedge ===')
|
||||
await runOccupancyDecisionPhase(primitives, scratch, staged)
|
||||
|
||||
log('=== PHASE 3: does the launcher actually hold on that verdict? ===')
|
||||
checkLauncherHoldsOccupiedDaemon({ log, assert })
|
||||
|
||||
verdict = 'PASS'
|
||||
} finally {
|
||||
for (const phase of staged) {
|
||||
teardown(phase)
|
||||
}
|
||||
rmSync(scratch, { recursive: true, force: true })
|
||||
|
||||
const survivors = foreignDaemons.filter((d) => isProcessAlive(d.pid))
|
||||
// Why only the real userData daemon is fatal: orphaned test daemons idle-shut-down or
|
||||
// death-watch out on their own schedule, so their exit during a 90s run proves nothing.
|
||||
const realUserDaemons = foreignDaemons.filter((d) => d.isRealUserDaemon)
|
||||
const harmedRealDaemons = realUserDaemons.filter((d) => !isProcessAlive(d.pid))
|
||||
const departed = foreignDaemons.filter((d) => !isProcessAlive(d.pid) && !d.isRealUserDaemon)
|
||||
const departedNote =
|
||||
departed.length > 0
|
||||
? ` (unrelated daemons that exited on their own: ${departed.map((d) => d.pid).join(', ')})`
|
||||
: ''
|
||||
log(
|
||||
`cleanup done; pre-existing daemons still running: ${survivors.map((d) => d.pid).join(', ') || 'none'}${departedNote}`
|
||||
)
|
||||
log(
|
||||
harmedRealDaemons.length > 0
|
||||
? `THE REAL userData DAEMON WAS HARMED: ${harmedRealDaemons.map((d) => d.pid).join(', ')}`
|
||||
: `real userData daemon untouched: ${realUserDaemons.map((d) => d.pid).join(', ') || 'none running'}`
|
||||
)
|
||||
if (harmedRealDaemons.length > 0) {
|
||||
verdict = 'FAIL'
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
`\n[daemon-pty-preservation] TIMELINE\n${timeline.map((line) => ` ${line}`).join('\n')}\n`
|
||||
)
|
||||
process.stdout.write(
|
||||
verdict === 'PASS'
|
||||
? '\n[daemon-pty-preservation] PASS: killStaleDaemon() on a wedged daemon still kills it and every agent PTY with it (phase 1); against the identical wedge resolveDaemonOccupancy() returns { occupied, liveSessions: null } from the process table with the daemon unsignalled, both agents alive, and the daemon recovering healthy with all sessions on SIGCONT (phase 2); and daemon-init.ts returns holdIncumbentDaemon() on that verdict, before any kill (phase 3, static).\n'
|
||||
: '\n[daemon-pty-preservation] FAIL: live agent PTYs are NOT protected — see the ERROR line and the timeline above.\n'
|
||||
)
|
||||
process.exitCode = verdict === 'PASS' ? 0 : 1
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`[daemon-pty-preservation] ERROR: ${error.stack ?? error.message}\n`)
|
||||
process.exitCode = 1
|
||||
})
|
||||
@@ -1,136 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Process-table helpers for the daemon PTY preservation repro: what is alive, what a pid is
|
||||
* running, and which daemons were already here before the run. Split out so the repro script
|
||||
* itself stays about the sequence it proves rather than the plumbing it proves it with.
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process'
|
||||
|
||||
// Electron's userData path differs per platform, and hardcoding the macOS one meant a real
|
||||
// daemon could never be recognised on Linux — so the guard that this run harmed nothing was
|
||||
// inert on exactly the platform where it would go unnoticed.
|
||||
const REAL_USER_DAEMON_MARKERS = {
|
||||
darwin: ['Library/Application Support/orca/daemon'],
|
||||
linux: ['.config/orca/daemon'],
|
||||
win32: ['AppData/Roaming/orca/daemon', 'AppData\\Roaming\\orca\\daemon']
|
||||
}
|
||||
const REAL_USER_DAEMON_MARKER_LIST = REAL_USER_DAEMON_MARKERS[process.platform] ?? ['orca/daemon']
|
||||
|
||||
export function processArgs(pid) {
|
||||
try {
|
||||
return execFileSync('ps', ['-p', String(pid), '-o', 'args='], {
|
||||
encoding: 'utf8',
|
||||
timeout: 5_000
|
||||
}).trim()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function processState(pid) {
|
||||
try {
|
||||
return execFileSync('ps', ['-p', String(pid), '-o', 'stat='], {
|
||||
encoding: 'utf8',
|
||||
timeout: 5_000
|
||||
}).trim()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function isProcessAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return true
|
||||
} catch (error) {
|
||||
return error?.code !== 'ESRCH'
|
||||
}
|
||||
}
|
||||
|
||||
// Why scan by tag rather than trust the session pid: macOS wraps the PTY in
|
||||
// /usr/bin/login for TCC attribution, so the agent process is a descendant of
|
||||
// the session leader — exactly as a real `claude`/`codex` launch would be.
|
||||
export function findTaggedPid(tag) {
|
||||
try {
|
||||
const output = execFileSync('ps', ['-eo', 'pid=,args='], {
|
||||
encoding: 'utf8',
|
||||
timeout: 5_000
|
||||
})
|
||||
for (const line of output.split('\n')) {
|
||||
if (line.includes(tag)) {
|
||||
const pid = Number(line.trim().split(/\s+/, 1)[0])
|
||||
if (Number.isInteger(pid) && pid > 0) {
|
||||
return pid
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ps failed; treat as not found.
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function isMarkerAlive(marker) {
|
||||
return processArgs(marker.pid)?.includes(marker.tag) === true
|
||||
}
|
||||
|
||||
/**
|
||||
* The session leader of a still-live marker, read now rather than remembered.
|
||||
*
|
||||
* Why not trust the pid captured at staging: teardown runs a minute later, and phase 1 has
|
||||
* deliberately killed things in between. A remembered leader pid may by then belong to whatever
|
||||
* the OS recycled it onto, and SIGKILLing that is precisely the mistake this script exists to
|
||||
* demonstrate. Returns null unless the live marker still claims this leader.
|
||||
*/
|
||||
export function verifiedSessionLeaderPid(marker) {
|
||||
if (!isMarkerAlive(marker)) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const ppid = Number(
|
||||
execFileSync('ps', ['-p', String(marker.pid), '-o', 'ppid='], {
|
||||
encoding: 'utf8',
|
||||
timeout: 5_000
|
||||
}).trim()
|
||||
)
|
||||
return Number.isInteger(ppid) && ppid === marker.sessionPid ? ppid : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-existing daemons (the user's real one above all) must be untouched by this run.
|
||||
export function snapshotForeignDaemons() {
|
||||
const daemons = []
|
||||
try {
|
||||
const output = execFileSync('ps', ['-eo', 'pid=,args='], { encoding: 'utf8', timeout: 5_000 })
|
||||
for (const line of output.split('\n')) {
|
||||
if (!line.includes('daemon-entry.js')) {
|
||||
continue
|
||||
}
|
||||
const pid = Number(line.trim().split(/\s+/, 1)[0])
|
||||
if (Number.isInteger(pid) && pid > 0) {
|
||||
daemons.push({
|
||||
pid,
|
||||
isRealUserDaemon: REAL_USER_DAEMON_MARKER_LIST.some((marker) => line.includes(marker))
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ps failed; the exit check will report an empty snapshot.
|
||||
}
|
||||
return daemons
|
||||
}
|
||||
|
||||
export async function waitFor(predicate, description, timeoutMs) {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
for (;;) {
|
||||
if (await predicate()) {
|
||||
return
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(`timed out waiting for ${description}`)
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 200))
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
const rawExtraArgs = process.argv.slice(2)
|
||||
const extraArgs = rawExtraArgs[0] === '--' ? rawExtraArgs.slice(1) : rawExtraArgs
|
||||
const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm'
|
||||
const env = {
|
||||
...process.env,
|
||||
ORCA_E2E_SSH_DOCKER: '1'
|
||||
}
|
||||
|
||||
// Why: Node's CVE-2024-27980 hardening rejects .cmd spawns without shell on Windows.
|
||||
const spawnOptions = {
|
||||
stdio: 'inherit',
|
||||
env,
|
||||
shell: process.platform === 'win32'
|
||||
}
|
||||
|
||||
const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], spawnOptions)
|
||||
|
||||
if (runtime.status !== 0) {
|
||||
process.exit(runtime.status ?? 1)
|
||||
}
|
||||
|
||||
const result = spawnSync(
|
||||
pnpm,
|
||||
[
|
||||
'exec',
|
||||
'playwright',
|
||||
'test',
|
||||
'tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts',
|
||||
'--config',
|
||||
'tests/playwright.config.ts',
|
||||
'--project',
|
||||
'electron-headless',
|
||||
'--workers=1',
|
||||
...extraArgs
|
||||
],
|
||||
spawnOptions
|
||||
)
|
||||
|
||||
process.exit(result.status ?? 1)
|
||||
@@ -1,41 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
const rawExtraArgs = process.argv.slice(2)
|
||||
const extraArgs = rawExtraArgs[0] === '--' ? rawExtraArgs.slice(1) : rawExtraArgs
|
||||
const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm'
|
||||
const env = {
|
||||
...process.env,
|
||||
ORCA_E2E_SSH_DOCKER: '1'
|
||||
}
|
||||
|
||||
// Why: Node's CVE-2024-27980 hardening rejects .cmd spawns without shell on Windows.
|
||||
const spawnOptions = {
|
||||
stdio: 'inherit',
|
||||
env,
|
||||
shell: process.platform === 'win32'
|
||||
}
|
||||
|
||||
const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], spawnOptions)
|
||||
|
||||
if (runtime.status !== 0) {
|
||||
process.exit(runtime.status ?? 1)
|
||||
}
|
||||
|
||||
const result = spawnSync(
|
||||
pnpm,
|
||||
[
|
||||
'exec',
|
||||
'playwright',
|
||||
'test',
|
||||
'tests/e2e/ssh-reconnect-pane-cardinality.spec.ts',
|
||||
'--config',
|
||||
'tests/playwright.config.ts',
|
||||
'--project',
|
||||
'electron-headless',
|
||||
'--workers=1',
|
||||
...extraArgs
|
||||
],
|
||||
spawnOptions
|
||||
)
|
||||
|
||||
process.exit(result.status ?? 1)
|
||||
@@ -1,41 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
const rawExtraArgs = process.argv.slice(2)
|
||||
const extraArgs = rawExtraArgs[0] === '--' ? rawExtraArgs.slice(1) : rawExtraArgs
|
||||
const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm'
|
||||
const env = {
|
||||
...process.env,
|
||||
ORCA_E2E_SSH_DOCKER: '1'
|
||||
}
|
||||
|
||||
// Why: Node's CVE-2024-27980 hardening rejects .cmd spawns without shell on Windows.
|
||||
const spawnOptions = {
|
||||
stdio: 'inherit',
|
||||
env,
|
||||
shell: process.platform === 'win32'
|
||||
}
|
||||
|
||||
const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], spawnOptions)
|
||||
|
||||
if (runtime.status !== 0) {
|
||||
process.exit(runtime.status ?? 1)
|
||||
}
|
||||
|
||||
const result = spawnSync(
|
||||
pnpm,
|
||||
[
|
||||
'exec',
|
||||
'playwright',
|
||||
'test',
|
||||
'tests/e2e/ssh-two-hosts-isolation.spec.ts',
|
||||
'--config',
|
||||
'tests/playwright.config.ts',
|
||||
'--project',
|
||||
'electron-headless',
|
||||
'--workers=1',
|
||||
...extraArgs
|
||||
],
|
||||
spawnOptions
|
||||
)
|
||||
|
||||
process.exit(result.status ?? 1)
|
||||
@@ -1,146 +0,0 @@
|
||||
# Terminal session behavior contract
|
||||
|
||||
Properties every terminal-session change must preserve. They describe
|
||||
observable behavior, not mechanism, so a test written against them stays valid
|
||||
under any implementation.
|
||||
|
||||
Oracles for the STA-3077 properties live in
|
||||
`src/main/ssh-reattach-pane-cardinality.test.ts`.
|
||||
|
||||
## Why this exists
|
||||
|
||||
Reconnecting an SSH-backed workspace used to add panes the user never opened,
|
||||
and the remote host accumulated shells nobody was using — one report went from
|
||||
2 to 19 to 20 relay PTYs across three reconnects. The failures in this class
|
||||
share one shape: **something absent was treated as something dead**, or a
|
||||
reattach was allowed to create what it should only have bound.
|
||||
|
||||
## A. Unknown is not dead
|
||||
|
||||
A disconnect, timeout, absent inventory entry, or failed inspection means
|
||||
_unresolved_. None of them may terminate a PTY, release a binding, or authorize
|
||||
cleanup.
|
||||
|
||||
**A5** — a timer may never be the _sole cause_ of a destructive action. A timer
|
||||
that bounds a wait or triggers re-verification is fine; a timer that kills is
|
||||
not. Retry count never decides identity or liveness.
|
||||
|
||||
This is deliberately weaker than "no timers". Recovery budgets and scratch-file
|
||||
age-gates are correct code and must stay.
|
||||
|
||||
Two patterns make a retention bound safe, and both beat shortening it:
|
||||
|
||||
- measure **process time**, not wall clock, so a suspended laptop does not
|
||||
burn the budget;
|
||||
- gate aggressive reclamation on an **independent observation** — a second
|
||||
consecutive scan, or another client completing a handshake — so the clock
|
||||
bounds a wait while evidence authorizes the act.
|
||||
|
||||
**B** — respawn requires proof. A failure that does not prove the session is
|
||||
gone is unresolved: leave the shell running and keep the binding. An
|
||||
unrecognized failure must never become a respawn, because a duplicated shell
|
||||
is worse than a stalled one.
|
||||
|
||||
## B. Attach binds; it never creates
|
||||
|
||||
An ambiguous, unavailable, or rejected attach does not spawn a shell and does
|
||||
not grow topology — no minted tab, no minted leaf, no split root, no minted
|
||||
layout. Only an explicit create for a fresh pane may spawn.
|
||||
|
||||
Corollary: after a recovery, keystrokes cannot land on a different shell than
|
||||
the one that was visible before it.
|
||||
|
||||
## C. Exact-binding fencing
|
||||
|
||||
Mutating operations name the binding captured before the first await and are
|
||||
rejected when any component no longer matches. A stale operation cannot reach a
|
||||
later pane generation, a later PTY incarnation reusing the same id, or a
|
||||
restarted owner. Output and exit arriving under a stale binding are dropped.
|
||||
|
||||
The store already enforces this via `persistPtyBinding`'s `expectedBinding`
|
||||
compare-and-swap and the per-repo topology revision fence.
|
||||
|
||||
## D. Cardinality
|
||||
|
||||
One pane owns at most one live PTY binding, and one PTY incarnation belongs to
|
||||
at most one pane. Lease identity must therefore include the pane, not just the
|
||||
transport id — keying on `(targetId, ptyId)` alone is what let predecessors
|
||||
accumulate.
|
||||
|
||||
Superseding a lease marks it expired; it does not kill the remote process,
|
||||
because losing a lease is not proof the shell died.
|
||||
|
||||
## E. Ordering and loss
|
||||
|
||||
Output is delivered in PTY order. A gap is resolved by resnapshot, never by
|
||||
silently continuing. A correctness-bearing effect (exit, command completion,
|
||||
review link, bell) is neither lost while the renderer is unmounted nor applied
|
||||
twice after a replay.
|
||||
|
||||
Bounded replay plus idempotent application by event key satisfies this. A
|
||||
durable per-consumer delivery cursor is not required, and is not currently
|
||||
used.
|
||||
|
||||
**Lifecycle is state, not an event.** An exit that happens while nothing is
|
||||
attached is not recovered from a delivery channel — it is read during the
|
||||
attach handshake. The attach reply should carry whether the session is alive
|
||||
or exited, its exit code, and the resume offset. That is what removes the need
|
||||
to guarantee delivery of an exit _event_, and it is why no journal is required
|
||||
here.
|
||||
|
||||
Where a resume offset cannot be served, advance the offset explicitly by the
|
||||
size of the gap and persist the running total, so "output was lost here" stays
|
||||
a queryable fact rather than a silent discontinuity.
|
||||
|
||||
## F. Isolation
|
||||
|
||||
A failure on one host cannot affect another host; a failure in one workspace
|
||||
cannot affect another on the same connection. There is no process-wide failure
|
||||
fence.
|
||||
|
||||
## G. Liveness
|
||||
|
||||
Every fail-closed path needs a bounded, user-reachable recovery. A workspace
|
||||
must never reach a state that permanently refuses terminal operations with no
|
||||
operator escape — in particular, backpressure from a consumer that is gone
|
||||
forever must be releasable without that consumer's cooperation.
|
||||
|
||||
Unavailable transports retry with bounded backoff and bounded memory.
|
||||
|
||||
## H. Compatibility
|
||||
|
||||
Clients and hosts update independently. A capability is negotiated or the newer
|
||||
side degrades; a required operation is never silently dropped. New stream
|
||||
opcodes must be capability-negotiated, because decoders drop unknown opcodes
|
||||
silently. Nothing is gated on an app build id.
|
||||
|
||||
See [`remote-wire-compatibility.md`](./remote-wire-compatibility.md).
|
||||
|
||||
## I. Cost
|
||||
|
||||
No new work on the input hot path and none per output frame. Pane create and
|
||||
close latency must not regress, including over a high-latency SSH link. No
|
||||
startup-blocking probe of a host nothing has asked for.
|
||||
|
||||
---
|
||||
|
||||
## Test the call site, not the capability
|
||||
|
||||
A guard that exists and is never passed is indistinguishable from no guard.
|
||||
`mayCreate` was added to `persistPtyBinding`, was correct, and had no
|
||||
production caller for several commits — every store-level test passed the whole
|
||||
time, because they called the store directly.
|
||||
|
||||
So for anything that refuses a destructive action, pin the **caller**: assert
|
||||
that the reattach path passes the refusal, not merely that the store honours it
|
||||
when asked. The same applies to a classifier — assert the branch that consumes
|
||||
its verdict is reachable, since a guard behind an unreachable `catch` is dead.
|
||||
|
||||
## Deliberately not required
|
||||
|
||||
A durable event journal, per-consumer cumulative delivery cursors, and
|
||||
cryptographic device principals are **not** required by any property above.
|
||||
Each would need to earn its place by being the only way to satisfy one of them.
|
||||
|
||||
Principals, in particular, exist only to key cursors: without cursors there is
|
||||
nothing for them to key.
|
||||
-192
@@ -1,192 +0,0 @@
|
||||
# Design: make "Start a new terminal" a replacement, not a spawn
|
||||
|
||||
Supersedes the mechanism in `PR-13326-SESSION-HANDOFF-2026-08-12.md` §"Approved bounded redesign".
|
||||
Keeps its **properties**; replaces several of its **mechanisms** with smaller ones, and adds the
|
||||
thing it was missing — a way to actually run the decisive test.
|
||||
|
||||
## 1. The defect, stated once
|
||||
|
||||
"Start a new terminal" is routed through a path whose job is *"resolve this pane's owner and attach
|
||||
it; spawn only if there is none."* Creation is that path's fallback, not its purpose. The pane's
|
||||
durable binding still names the old shell at click time, so the path resolves it and adopts it.
|
||||
|
||||
I confirmed this first-hand earlier in the branch: commit `ab6ce176f3f` detects adoption after the
|
||||
fact (`freshPtyId === sessionId`) and stops the callback from unbinding the live shell. Its own
|
||||
message records that suppressing the adoption itself was left undone. So the adoption is real, and
|
||||
the current code only contains its worst consequence.
|
||||
|
||||
## 2. Why every previous fix leaked
|
||||
|
||||
Each attempt expressed "do not adopt" as **another condition inside the adopting path**. Conditions
|
||||
are forgettable and order-dependent, and this branch has three shipped proofs of that failure mode:
|
||||
|
||||
- a guard centralised into a publisher that returned `void`, so four callers paired a conditional
|
||||
publish with an unconditional `return`;
|
||||
- the same rule applied to the tombstone branch and missed on the liveness branch beside it;
|
||||
- the incarnation fence sent on the pane-driven route and not on the reconnect route.
|
||||
|
||||
Every one was "the rule exists, but this path does not ask it." A fourth condition in the same path
|
||||
is the same bet again.
|
||||
|
||||
## 3. The design, and why it removes the defect
|
||||
|
||||
### P1 — Replacement is a separate operation that cannot adopt
|
||||
|
||||
Add a main-owned entry that performs replacement only. It contains **no owner-resolution call**: it
|
||||
does not read a stable-pane owner, does not accept `sessionId`/`attachOnly`, and constructs provider
|
||||
options from a narrow main-owned allowlist (dimensions + validated pane/workspace identity).
|
||||
|
||||
*Why this fixes it:* adoption stops being a branch that must be avoided and becomes code that is not
|
||||
present. A function with no owner-resolution cannot adopt for the same reason a function with no
|
||||
socket cannot make a network call. This converts "remember to skip adoption" (liveness) into "there
|
||||
is nothing here to adopt with" (structure).
|
||||
|
||||
*Falsifiable by:* route the action back through generic spawn — the oracle in P0 must redden.
|
||||
|
||||
### P2 — Ownership moves in one durable commit, keyed on exact incarnation
|
||||
|
||||
One Store method mutates and flushes once: exact-CAS the old binding and the old active lease,
|
||||
install the new binding + incarnation + attached lease, expire the exact old lease **without
|
||||
touching its process**, bump the topology/lease versions, flush.
|
||||
|
||||
*Why:* ownership lives in the durable binding and the lease. Moved separately, there is a window
|
||||
where a racing reconnect sees a half-moved pane and restores the predecessor. One commit means the
|
||||
window does not exist rather than being narrow.
|
||||
|
||||
*Why exact incarnation:* the relay reissues `pty-N` from 1 after a reset, so any CAS keyed on ptyId
|
||||
alone can match a different process. This branch already proved that and already carries the
|
||||
primitive: host-attested `SshRemotePtyLease.incarnationId`, the relay-side `expectedIncarnationId`
|
||||
fence, and `isRelayAttestedPtyIncarnationId` filtering synthesised values.
|
||||
|
||||
*Falsifiable by:* split the commit in two flushes and drive a reconnect between them.
|
||||
|
||||
### P3 — Ambiguity leaks, never kills
|
||||
|
||||
Provider/relay shutdown identifies a PTY by id only. After a dispatched spawn whose outcome is
|
||||
ambiguous, we cannot prove which incarnation a shutdown would reach. So: clean up a failed new shell
|
||||
only when its exact incarnation is provable; otherwise leave it running and unbound for the existing
|
||||
cleanup surface to reap.
|
||||
|
||||
*Why:* the costs are asymmetric. Leaking costs an orphan shell that a user can already list and
|
||||
terminate. Killing the wrong incarnation destroys a live agent session — the exact harm this whole
|
||||
program exists to prevent.
|
||||
|
||||
*Falsifiable by:* make the ambiguous path call shutdown; the oracle asserting zero kill RPCs on an
|
||||
unprovable outcome must redden.
|
||||
|
||||
### P4 — The successor is invisible until it is the owner
|
||||
|
||||
Spawn the replacement hidden and install its output handlers only after P2 commits.
|
||||
|
||||
*Why:* a new shell reusing the old raw id could otherwise deliver bytes into the predecessor's
|
||||
model. Ordering removes the window without a new quarantine subsystem.
|
||||
|
||||
### P5 — Stale renderer work does nothing
|
||||
|
||||
Guard every completion on: not disposed, current transport identity, current action generation. A
|
||||
stale callback must not clear a binding, republish the card, or kill a result. Main owns the durable
|
||||
transition, so the renderer never clears the old binding on success.
|
||||
|
||||
## 4. What I am deliberately NOT building, and the risk of each
|
||||
|
||||
The handoff prescribes more machinery. I judge these unnecessary for this defect; each is a
|
||||
reversible decision if an oracle shows otherwise.
|
||||
|
||||
| Prescribed | Decision | Reasoning | Risk if I am wrong |
|
||||
|---|---|---|---|
|
||||
| Opaque ticket bound to ~11 fields (sender, both generations, worktree, tab, leaf, pane key, both ids, both incarnations, expiry, 5-state machine) | **Replace** with an operation id + expected-owner `{ptyId, incarnationId}` CAS | The renderer is not a trust boundary — it can already open terminals. The ticket's real jobs are staleness and idempotency, which the CAS and op id give directly. Eleven bindings add eleven invalidation paths, and invalidation bugs are the same class we are trying to leave | A forged/stale request creates one blank shell for a pane the user owns — recoverable, and the CAS still refuses to move ownership off an unexpected incumbent |
|
||||
| A new shared owner mutex | **Reuse** `paneSpawnReservationsByOwnerKey` (already per-pane) and the runtime pane-create claim | Serialisation already exists at exactly this granularity; a second lock invites lock-order bugs | Two operations interleave on one pane; the P2 CAS still refuses the loser |
|
||||
| A quarantine subsystem for provider output | **Reuse** hidden spawn + install-after-commit ordering | Achieves the same property with existing options | Bytes reach the wrong model; caught by the P4 oracle |
|
||||
|
||||
## 5. The part the previous session was missing: the decisive test can now run
|
||||
|
||||
`tests/e2e/ssh-disconnected-pane-affordance.spec.ts` holds the decisive case at `test.fixme` because
|
||||
**the state was not inducible**: it needs the SSH target CONNECTED while exactly one pane's
|
||||
`pty.attach` fails with an error that does not prove the shell gone. Host faults tried (SIGSTOP the
|
||||
relay) take the whole connection down and produce the connection overlay instead.
|
||||
|
||||
That state is now inducible, by a mechanism this branch added for another reason. The relay refuses
|
||||
an attach whose `expectedIncarnationId` names a different shell, with
|
||||
`PTY "<id>" identity mismatch` — per-pty, connection healthy, and explicitly not proof of death
|
||||
(`SSH_PTY_IDENTITY_MISMATCH` never grounds a respawn).
|
||||
|
||||
So the test can seed the pane's lease with an incarnation that is not the host's, reconnect, and get
|
||||
exactly the state the banner needs — deterministically, with no timing fault.
|
||||
|
||||
**This is the highest-value item in the plan and it comes first.** The previous session looped
|
||||
because every claim was adjudicated by reading code; with the oracle running, the design is
|
||||
*demonstrated* instead of argued, and a regression reddens instead of being found in review.
|
||||
|
||||
## 6. Goalposts
|
||||
|
||||
Each is a behaviour with a stated mutation that must redden it. G1 is a prerequisite for believing
|
||||
any of the others.
|
||||
|
||||
| # | Goalpost | Mutation that must redden it |
|
||||
|---|---|---|
|
||||
| **G1** | The decisive E2E runs unskipped: with the target connected, one pane unreachable, clicking "Start a new terminal" yields exactly one new remote shell, the pane count is unchanged, and the host transcript shows no second agent launch | Remove the induction; the test can no longer reach the banner |
|
||||
| **G2** | The action never adopts: the pane's shell id/incarnation after the click differs from before, and the new shell has no command, resume metadata, launch token, agent, or startup delivery | Route the action back through generic `pty:spawn` |
|
||||
| **G3** | Ownership moves atomically: a reconnect racing the replacement cannot restore the predecessor; binding and lease agree at every observable point | Split the durable commit into two flushes |
|
||||
| **G4** | Ambiguity never kills: on an unprovable spawn outcome, zero shutdown RPCs are issued and neither process dies | Call ID-only shutdown on the ambiguous path |
|
||||
| **G5** | The successor is silent until it owns the pane: no provider bytes reach the predecessor's model, including when the raw id is reused | Install output handlers before the commit |
|
||||
| **G6** | Stale renderer work is inert: a disposed or superseded callback clears no binding, republishes no card, kills no result | Drop the generation guard on the completion path |
|
||||
|
||||
## 7. Sequence
|
||||
|
||||
0. Remove the rejected six-file WIP (targeted edits, not a broad restore — the worktree is dirty).
|
||||
1. **G1** — induction + un-`fixme`. Nothing else is believable until this runs.
|
||||
2. **G2** — the separate replacement operation.
|
||||
3. **G3** — the single durable commit.
|
||||
4. **G4/G5/G6** — ambiguity, ordering, staleness.
|
||||
5. Adversarial review on both models, then push.
|
||||
|
||||
## 8. Stop conditions
|
||||
|
||||
- If G1 cannot be made to run, stop and report. Building 2–6 without it repeats the previous loop.
|
||||
- If exact cleanup cannot be achieved within existing local/provider contracts, take the leak (P3).
|
||||
Do **not** add `expectedIncarnationId` to remote shutdown — that is a separate, capability-
|
||||
negotiated wire change with its own review.
|
||||
|
||||
---
|
||||
|
||||
## 9. Findings from running the gate (2026-08-13)
|
||||
|
||||
The oracle in §5 works: the state is inducible and the card appears reliably. Running it found
|
||||
three things that reading had not.
|
||||
|
||||
**Fixed — the card's buttons were unclickable.** `TerminalErrorToast` renders at `z-50` in the same
|
||||
bottom strip and was suppressed only for the connection overlay, not for the pane's own card. In
|
||||
the one state this affordance exists for, both actions were covered. `TerminalPane.tsx` now also
|
||||
suppresses it while the active pane shows the card.
|
||||
|
||||
**Fixed — a session id is the instruction to attach.** "Start a new terminal" was still sending the
|
||||
pane's recorded `sessionId`, which routes straight to reattach before any pane-owner logic runs, so
|
||||
the action attached the very shell it could not reach and created nothing. Skipping owner
|
||||
resolution in main was NOT enough on its own; the id had to be dropped at the last gate before the
|
||||
IPC (`admittedSessionId` in `pty-transport.ts`).
|
||||
|
||||
**OPEN — the action produces no spawn at all.** With both fixes in, the measured result after
|
||||
clicking is still `visible=true launches=1 shells=1`: the card clears briefly, no shell is created,
|
||||
and the card returns. The main log over the same window shows only the pane's own restore retries
|
||||
(three `spawn() called with sessionId=…` → `identity mismatch`), never a fresh spawn.
|
||||
|
||||
The evidence points at the action closure rather than the spawn path: the pane retries its restore
|
||||
while the card is up, each retry republishes the card, and the button the test clicks appears to
|
||||
belong to a superseded connection whose `startFreshSpawn` no longer does anything. That is the
|
||||
"stale renderer work" hazard from the original handoff, showing up as a dead button rather than as
|
||||
a wrong write.
|
||||
|
||||
**RESOLVED.** Instrumenting handler, transport and main in one correlated run disproved the
|
||||
closure hypothesis: the handler ran, the connection was live, and the renderer half was already
|
||||
correct — it sent no session id and asked for adoption to be refused. Main re-derived the id anyway.
|
||||
|
||||
The rule had been applied at the two places an owner is PRODUCED and missed at the one place it is
|
||||
CONSUMED: `spawnForStablePane` turns an owner into `sessionId` for the provider, which is what makes
|
||||
an attach an attach. Gating there — `if (args.owner && !args.refuseAdoption)` — closes it for every
|
||||
producer at once, present and future.
|
||||
|
||||
The gate now passes, and reverting that single condition reddens it.
|
||||
|
||||
Lesson worth keeping: this rule leaked at three sites in a row (early owner resolution, the
|
||||
transport's session id, the owner consumption). Each fix was necessary and none was sufficient. The
|
||||
one that finally held was the one placed where the value is USED rather than where it is derived.
|
||||
-823
@@ -1,823 +0,0 @@
|
||||
# STA-3077 / PR #13326 session handoff
|
||||
|
||||
> **Status:** paused at an approved bounded redesign; not release-ready.
|
||||
>
|
||||
> **Review/fix round:** 6.
|
||||
>
|
||||
> **Do not continue the current six-file implementation.** It is a rejected,
|
||||
> uncommitted grant-map approach. Remove it with `apply_patch`, then implement
|
||||
> the dedicated main-owned replacement transaction described below.
|
||||
|
||||
This is the continuation entrypoint for the current PR. It records the whole
|
||||
session at the point the user asked to pause, including what shipped to the PR
|
||||
branch, what failed review, the agreed design boundary, and the next safe work.
|
||||
|
||||
## Executive summary
|
||||
|
||||
The session started with STA-3077's SSH reconnect failure class: reconnects
|
||||
could graft panes the user never created, accumulate remote shells, fabricate
|
||||
process death from an attach failure, or let stale identities affect a
|
||||
successor. The branch now contains extensive fixes, tests, design records, and
|
||||
two user-visible recovery actions for an unreachable pane.
|
||||
|
||||
The pushed candidate is still incorrect in one important action. “Start a new
|
||||
terminal” invokes generic `pty:spawn` while the old durable pane binding remains.
|
||||
Main can adopt that old owner, so the action can reopen the old shell instead of
|
||||
creating the one blank shell the user requested.
|
||||
|
||||
An attempted six-file fix added `freshSpawnForUnreachablePtyId` to generic spawn
|
||||
and a process-global grant map. Three independent read-only audits rejected it.
|
||||
The common verdict was that replacement must be a dedicated local transaction
|
||||
owned by Electron main, with exact incarnation identity, one durable Store
|
||||
commit, runtime ownership transfer, output quarantine, reconnect fencing, and
|
||||
renderer generation guards.
|
||||
|
||||
The user approved this bounded redesign. It applies only to the broken
|
||||
unreachable-pane replacement operation. It is not authority to restart the
|
||||
earlier whole-system architecture effort.
|
||||
|
||||
## Repository checkpoint
|
||||
|
||||
Verified before this handoff was written:
|
||||
|
||||
| Item | Value |
|
||||
| ----------------------------------- | ---------------------------------------------------- |
|
||||
| Worktree | `/Users/nwparker/orca/workspaces/orca/eye-React-185` |
|
||||
| Local branch | `nwparker/sta-3077-reattach-pane-cardinality` |
|
||||
| HEAD | `df6edfcebd2ea401a6feda1fccf31189e57eceda` |
|
||||
| HEAD subject | `fix(ui): center narrow terminal recovery actions` |
|
||||
| Actual PR | `stablyai/orca#13326` |
|
||||
| Actual PR ref | `origin/nwparker/sta-3077-d-affordance-and-review` |
|
||||
| Actual PR ref SHA | exactly `df6edfcebd2ea401a6feda1fccf31189e57eceda` |
|
||||
| Configured upstream | `origin/nwparker/sta-3077-reattach-pane-cardinality` |
|
||||
| Configured upstream SHA | `518dac34415e0f67883461c3f58e5586209c7f24` |
|
||||
| Local `origin/main` | `889dd4c7d2fe255aac552e4dcc91204279dc022c` |
|
||||
| Merge base with local `origin/main` | `58a926170cd99baf8920a83a86799bfbd307d735` |
|
||||
| Local `origin/main...HEAD` | 10 commits on main / 109 commits on HEAD |
|
||||
|
||||
The configured upstream is stale. Its `[ahead 387]` status is not the PR's
|
||||
push state. The PR ref is exactly at HEAD. When the work is genuinely ready,
|
||||
push explicitly:
|
||||
|
||||
```bash
|
||||
git push origin HEAD:nwparker/sta-3077-d-affordance-and-review
|
||||
```
|
||||
|
||||
Before this document, the worktree had six modified tracked files, no untracked
|
||||
files, and a `+163/-11` uncommitted diff. `git diff --check` was clean. No
|
||||
product test, typecheck, lint, build, Electron, or E2E command was run against
|
||||
that dirty state.
|
||||
|
||||
## The intended outcome
|
||||
|
||||
For a pane whose old SSH terminal cannot currently be reached, the UI offers:
|
||||
|
||||
- **Try again:** attempt to reattach the exact existing shell without claiming
|
||||
it exited or spawning a replacement.
|
||||
- **Start a new terminal:** keep the same pane and leaf, create exactly one
|
||||
genuinely blank shell, move durable and runtime pane ownership to it, and
|
||||
leave the unproven old process alive but unbound.
|
||||
|
||||
The second action must never:
|
||||
|
||||
- silently adopt or resume the old shell;
|
||||
- resume an agent, command, launch token, startup sequence, or saved telemetry;
|
||||
- create another pane or tab;
|
||||
- kill a process whose exact incarnation is not proven;
|
||||
- let a reconnect restore the predecessor after replacement wins;
|
||||
- let stale renderer work clear, kill, or republish the successor;
|
||||
- expose provider output before the replacement commits; or
|
||||
- add replacement intent to SSH, relay, provider, paired-runtime, or stream
|
||||
wire contracts.
|
||||
|
||||
## What was done during the session
|
||||
|
||||
The branch has 109 first-parent commits over its current merge base, including
|
||||
two large updates from `origin/main`. The useful progression is below; use the
|
||||
git history for the complete commit-by-commit record.
|
||||
|
||||
### 1. Reconnect and persistence containment
|
||||
|
||||
The early work stopped SSH reconnect from treating persisted rows as permission
|
||||
to create UI, made pane binding authoritative during duplicate arbitration,
|
||||
and added rollback around durable lease retirement.
|
||||
|
||||
Representative commits:
|
||||
|
||||
- `5d8bb18fd3f` — stop reconnect from grafting panes and stacking leases.
|
||||
- `ae134995c13` — heal duplicate pane leases from older state.
|
||||
- `ccb082740fc` — let the durable pane binding outrank recency.
|
||||
- `8d9aeb55e03` — roll back lease retirement when durable writing fails.
|
||||
- `03d794a0f90` — pass `mayCreate: false` from the production reattach writer.
|
||||
- `a7558a837ca`, `13ca7630116`, and `8ec025e4e46` — add and strengthen pane,
|
||||
process, binding, and lease census oracles.
|
||||
|
||||
### 2. Unknown is not death
|
||||
|
||||
The session removed several paths that inferred a dead process from missing,
|
||||
unavailable, timed-out, or disconnected state.
|
||||
|
||||
Representative commits:
|
||||
|
||||
- `eec807d7594` — stop respawning a shell that may still be running.
|
||||
- `356d6b52b24` — apply positive-proof requirements to both reattach paths.
|
||||
- `b9d58423121` and `7589df8b348` — let PTY liveness remain unknown.
|
||||
- `7cd7fef927f` — a reattach not-found is not itself proof of shell death.
|
||||
- `e39514e9ab2` and `41613b5c84c` — distinguish bare not-found from a relay-
|
||||
proven exit and retire a lease only in the latter case.
|
||||
|
||||
This work also stopped one exhausted delivery recovery from tearing down every
|
||||
session on an SSH host (`07e382ec196`) and made parked delivery recoveries
|
||||
expire/re-enter correctly (`124e00e8a83`, `601604d17b1`).
|
||||
|
||||
### 3. Exact operation and incarnation fencing
|
||||
|
||||
The branch added or strengthened stale-operation refusal, host-attested shell
|
||||
identity, remembered-exit identity, and reconnect fences.
|
||||
|
||||
Representative commits:
|
||||
|
||||
- `0de5994be91` and `980e5d846b5` — fence stale writes, resize, and signals.
|
||||
- `235e45ad0f2` — record host-attested shell identity on a lease.
|
||||
- `30441f1ebbb` — fence recycled relay PTY IDs by shell identity.
|
||||
- `8f0c8e3ed35`, `42cbe34523f`, `c14ba64d9dc`, and `0cb187f6035` — require an
|
||||
exit proof to name the shell for which it grants replacement authority.
|
||||
- `5793f186e96`, `be02c4e7f5c`, and `901dd06b8d1` — apply incarnation and pane
|
||||
fences on reconnect, including clients unable to name a shell.
|
||||
- `c8d08e1b215` — strengthen the incarnation write-fence regression oracle.
|
||||
|
||||
These improvements are valuable, but the round-6 review found that raw PTY ID
|
||||
is still used as identity in important lease/runtime/cleanup paths. The new
|
||||
replacement transaction must finish that exact-incarnation work for its own
|
||||
surface rather than assuming the existing fencing is complete.
|
||||
|
||||
### 4. Pane arbitration and rollback hardening
|
||||
|
||||
Lease arbitration moved from tab-level assumptions toward the actual pane leaf,
|
||||
and rollback was hardened so an already-expired duplicate cannot be revived.
|
||||
|
||||
Representative commits:
|
||||
|
||||
- `b92498468a5` — arbitrate on the pane, not the tab.
|
||||
- `1697b3be163` — supersede on the leaf.
|
||||
- `7724a950ecd` — mutate only the plane that supplied the supersession proof.
|
||||
- `21ad47c0bc5` — fence duplicate-lease rollback and add a regression oracle.
|
||||
|
||||
The existing `upsertSshRemotePtyLease` key remains `(targetId, ptyId)`. That is
|
||||
insufficient when an old and a new process reuse the same raw ID but have
|
||||
different incarnations. Replacement requires exact lease identity.
|
||||
|
||||
### 5. Unreachable-pane product behavior
|
||||
|
||||
The renderer now presents a provider-neutral disconnected card with two
|
||||
explicit actions and copy that does not claim an unobserved exit.
|
||||
|
||||
Representative commits:
|
||||
|
||||
- `5e1b57bb2f9` — add the two-action disconnected pane affordance.
|
||||
- `d27927b84b5` — remove a remote-host-only assumption from its copy.
|
||||
- `037ee535b8a` — centralize unreachable-pane guards.
|
||||
- `f7eb6f26656` — suppress the entire saved startup set for a fresh shell.
|
||||
- `b46a1f8773e` — avoid stranding a local pane after restore failure.
|
||||
- `ab6ce176f3f` — do not unbind a live shell that generic spawn adopted.
|
||||
- `df6edfcebd2` — center the recovery actions in narrow panes.
|
||||
|
||||
The last item in that list makes the pushed behavior internally safer but also
|
||||
exposes the unresolved product bug: generic spawn can adopt the incumbent, so
|
||||
“Start a new terminal” is not guaranteed to start anything new.
|
||||
|
||||
### 6. Tests, design records, and platform evidence
|
||||
|
||||
The session added substantial focused and E2E coverage for local restart,
|
||||
daemon/WSL survival, SSH pane cardinality, two-host isolation, `MaxSessions=1`,
|
||||
mixed-version decoding, lease termination, relay exit proof, and renderer
|
||||
recovery behavior. The main records are:
|
||||
|
||||
- `docs/reference/terminal-session-behavior-contract.md`
|
||||
- this folder's `goalposts.md`, `design-final-detail.md`, and `resume-plan.md`
|
||||
- `src/main/ipc/pty.test.ts`
|
||||
- `src/main/ipc/pty-superseded-operation-fence.test.ts`
|
||||
- `src/main/persistence.test.ts`
|
||||
- `src/main/persistence-ssh-lease-termination.test.ts`
|
||||
- `src/main/runtime/orca-runtime.test.ts`
|
||||
- `src/main/ssh-reattach-pane-cardinality.test.ts`
|
||||
- `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts`
|
||||
- `src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts`
|
||||
- `src/renderer/src/components/terminal-pane/pty-connection.test.ts`
|
||||
- `src/renderer/src/components/terminal-pane/pty-transport.test.ts`
|
||||
- `src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx`
|
||||
- `tests/e2e/ssh-disconnected-pane-affordance.spec.ts`
|
||||
|
||||
Historical receipts in `goalposts.md` include discriminating native macOS,
|
||||
Linux, Windows, physical WSL, daemon, and Docker work at earlier candidate
|
||||
SHAs. They are historical evidence, not a current-HEAD validation receipt.
|
||||
|
||||
Important proof corrections were also recorded during the session:
|
||||
|
||||
- a guard tested only through a direct Store call can still have no production
|
||||
caller;
|
||||
- a green E2E that also passes without the fix is a forward guard, not proof;
|
||||
- the disconnected-pane state is not currently inducible with the existing
|
||||
Docker host faults; and
|
||||
- the first recovery grant and a later pane-binding fold were deleted after
|
||||
review showed their premises were false or their path was unreachable.
|
||||
|
||||
## The proven current bug
|
||||
|
||||
At pushed HEAD, the renderer's disconnected action calls `startFreshSpawn`,
|
||||
which calls the normal transport `connect`, which invokes generic `pty:spawn`.
|
||||
The old durable binding is still present. Generic spawn resolves stable pane
|
||||
ownership and may attach/adopt that incumbent rather than create a blank shell.
|
||||
|
||||
Consequences:
|
||||
|
||||
- “Start a new terminal” can reopen the old terminal.
|
||||
- A same-ID/new-incarnation result is not representable in the current renderer
|
||||
result contract.
|
||||
- The action's success path may compare or clear ownership by raw PTY ID.
|
||||
- The full product oracle is not running.
|
||||
|
||||
The decisive E2E is
|
||||
`tests/e2e/ssh-disconnected-pane-affordance.spec.ts`, test “starting a new
|
||||
terminal adds exactly one shell and leaves the panes alone,” beginning near line 314. It is still unconditionally held by
|
||||
`test.fixme(true, UNREACHABLE_PANE_INDUCTION_UNAVAILABLE)` near line 318.
|
||||
|
||||
## Dirty WIP that must be removed
|
||||
|
||||
The six uncommitted files implement the rejected
|
||||
`freshSpawnForUnreachablePtyId`/grant-map design:
|
||||
|
||||
| File | Uncommitted behavior |
|
||||
| ------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `src/main/ipc/pty.ts` | Adds a max-256 process-global grant map, records a raw `StablePaneOwner` after attach failure, accepts replacement intent on generic spawn, skips adoption, and performs best-effort raw-ID cleanup. |
|
||||
| `src/preload/api-types.ts` | Adds `freshSpawnForUnreachablePtyId` to generic spawn. |
|
||||
| `src/preload/index.ts` | Forwards that generic spawn option. |
|
||||
| `src/renderer/src/components/terminal-pane/pty-connection.ts` | Passes the old `sessionId` as replacement authority. |
|
||||
| `src/renderer/src/components/terminal-pane/pty-transport-types.ts` | Adds the option to the generic transport contract. |
|
||||
| `src/renderer/src/components/terminal-pane/pty-transport.ts` | Forwards it to generic spawn and suppresses telemetry with saved startup. |
|
||||
|
||||
Why it was rejected:
|
||||
|
||||
1. Raw PTY ID is not process identity; same-ID/different-incarnation leases and
|
||||
runtime owners collapse.
|
||||
2. Renderer input becomes replacement authority on a broad generic IPC.
|
||||
3. Grants have no TTL, sender binding, one-shot operation state, reload
|
||||
invalidation, or post-spawn revalidation.
|
||||
4. Concurrent pane reservations can return another operation's result rather
|
||||
than re-resolving the caller's intent.
|
||||
5. Binding and lease changes flush separately, so no atomic replacement exists.
|
||||
6. Provider output can reach the old model before persistence/runtime commit.
|
||||
7. Generic spawn still accepts renderer-derived environment and other startup
|
||||
data, so it is not a guaranteed blank shell.
|
||||
8. Runtime pane ownership is registered, not exactly transferred; lookup can
|
||||
continue returning the old record.
|
||||
9. Renderer callbacks remain authoritative after dispose or a newer attempt.
|
||||
10. Failure cleanup calls ID-only shutdown and can kill the wrong incarnation.
|
||||
11. The grant is minted on only one attach-failure path, while the UI action can
|
||||
be published from other paths, leaving a visible dead action.
|
||||
12. Replacement intent leaks into `PtySpawnOptions`-adjacent contracts, making
|
||||
remote-wire and mixed-version reasoning unnecessarily dangerous.
|
||||
|
||||
No uncommitted tests cover this WIP, and no test was run against it.
|
||||
|
||||
Remove only these WIP edits with `apply_patch`. Do not use `git reset`,
|
||||
`git checkout --`, or another broad restoration command in this dirty worktree.
|
||||
|
||||
## Round-6 independent review result
|
||||
|
||||
Three read-only reviews converged:
|
||||
|
||||
- **Relay/wire review:** request changes. It identified raw-ID identity,
|
||||
ambiguous cleanup, missing atomic Store commit, precommit output publication,
|
||||
broad IPC authority, incomplete blank-shell enforcement, and wire-
|
||||
compatibility blockers.
|
||||
- **Targeted transaction review:** recommended a dedicated IPC, owner-keyed
|
||||
mutex, token-keyed idempotency, combined Store transaction, reconnect CAS,
|
||||
post-image inspection on flush error, and deterministic race tests.
|
||||
- **Final readiness audit:** request changes. It independently found the same
|
||||
persistence/runtime/grant/incarnation/renderer/cleanup defects and one hard
|
||||
feasibility boundary: existing SSH shutdown is raw-ID-only.
|
||||
|
||||
All three reviews were read-only. They ran no tests or builds and changed no
|
||||
files.
|
||||
|
||||
## Approved bounded redesign
|
||||
|
||||
### 1. Main-issued replacement ticket
|
||||
|
||||
When main classifies an attach as unreachable but not proven dead, return a
|
||||
typed local result containing an opaque ticket. The ticket is authority, not a
|
||||
renderer-supplied PTY ID.
|
||||
|
||||
Bind it to:
|
||||
|
||||
- the issuing `webContents`/sender;
|
||||
- connection and provider generation;
|
||||
- worktree or folder-workspace identity;
|
||||
- tab, leaf, and pane key;
|
||||
- exact old application PTY ID and provider-native ID;
|
||||
- persisted and runtime incarnation IDs;
|
||||
- exact active lease identity;
|
||||
- issuance generation and short expiry;
|
||||
- a random operation ID; and
|
||||
- state `issued | running | committed | rejected | unresolved` plus the
|
||||
idempotent result where applicable.
|
||||
|
||||
Invalidate tickets on renderer reload/destruction, provider replacement,
|
||||
connection re-registration, expiry, or exact-owner change. A consumed or
|
||||
rejected ticket must not become valid again after any request was dispatched.
|
||||
|
||||
### 2. Dedicated local IPC
|
||||
|
||||
Add a local-only API such as:
|
||||
|
||||
```ts
|
||||
pty.replaceUnreachablePane({ ticketId, cols, rows })
|
||||
```
|
||||
|
||||
It should return a structured result such as:
|
||||
|
||||
```ts
|
||||
{
|
||||
;(ptyId, incarnationId, operationId)
|
||||
}
|
||||
```
|
||||
|
||||
It must not route through generic `pty:spawn` or `PtyTransport.connect`, and it
|
||||
must not add fields to provider, SSH, relay, paired-runtime, or stream payloads.
|
||||
Authenticate the sender as the main renderer window before ticket lookup.
|
||||
|
||||
### 3. Shared owner lock and idempotency
|
||||
|
||||
Use one host/worktree/pane owner mutex for:
|
||||
|
||||
- ordinary pane spawn/adoption;
|
||||
- unreachable-pane replacement; and
|
||||
- relay reconnect's final owner activation.
|
||||
|
||||
Each waiter must re-resolve its own intent after acquiring the lock. It may not
|
||||
return another operation's result merely because it waited on the same lock.
|
||||
|
||||
Use the ticket's operation ID for idempotency. A double-click, lost response,
|
||||
or retry while running awaits the same operation and returns the same result.
|
||||
It must create at most one shell.
|
||||
|
||||
### 4. Main-constructed blank spawn
|
||||
|
||||
Under the owner lock, re-read the workspace and re-derive CWD. Repeat folder-
|
||||
workspace path validation at that moment; do not trust the path captured when
|
||||
the ticket was issued.
|
||||
|
||||
Construct provider spawn options from a narrow main-owned allowlist. Exclude:
|
||||
|
||||
- command and command delivery;
|
||||
- `sessionId`, `attachOnly`, or resume metadata;
|
||||
- launch config, token, agent, or telemetry;
|
||||
- arbitrary renderer environment or environment deletion;
|
||||
- shell override;
|
||||
- startup command delivery; and
|
||||
- sequenced-startup environment.
|
||||
|
||||
Only the validated pane/workspace identity, dimensions, and provider-required
|
||||
blank-shell metadata may cross the provider boundary.
|
||||
|
||||
### 5. Exact incarnation lease identity
|
||||
|
||||
Lease identity for this transaction must include incarnation, not only
|
||||
`(targetId, ptyId)`. Old `pty-1/inc-A` and new `pty-1/inc-B` must coexist as:
|
||||
|
||||
- old exact lease: expired/unbound; and
|
||||
- new exact lease: attached/bound.
|
||||
|
||||
Every mark, remove, attach, rollback, and reconnect CAS touched by replacement
|
||||
must name the exact lease incarnation. Preserve compatibility with persisted
|
||||
legacy rows that lack optional incarnation data; unknown identity may not
|
||||
authorize destructive cleanup.
|
||||
|
||||
### 6. One Store transaction
|
||||
|
||||
Add one Store method, for example `replaceUnreachableSshPaneOwner`, that under
|
||||
one mutation and one flush:
|
||||
|
||||
1. exact-CASes the old durable pane binding;
|
||||
2. exact-CASes the old active lease;
|
||||
3. installs the new binding and incarnation;
|
||||
4. installs the new exact attached lease;
|
||||
5. expires the exact old lease without shutting down its process;
|
||||
6. advances topology and lease mutation versions, including affected already-
|
||||
expired siblings; and
|
||||
7. flushes once.
|
||||
|
||||
Snapshot every mutated in-memory field before starting.
|
||||
|
||||
Because durable-file replacement renames the new image before directory fsync,
|
||||
a thrown flush is not proof that disk stayed old. Inspect the persisted post-
|
||||
image before deciding what happened:
|
||||
|
||||
- **Old image:** restore memory and clean only a provably exact new process.
|
||||
- **New image:** treat the transaction as committed.
|
||||
- **Mixed or unreadable:** kill neither process; mark the operation unresolved,
|
||||
quarantine publication, and reconcile from disk/provider evidence.
|
||||
|
||||
### 7. Runtime ownership replacement
|
||||
|
||||
Add a non-exit runtime primitive along these lines:
|
||||
|
||||
```ts
|
||||
runtime.replaceTerminalPaneOwner({
|
||||
paneKey,
|
||||
worktreeId,
|
||||
expected: { ptyId, incarnationId },
|
||||
replacement: { ptyId, incarnationId }
|
||||
})
|
||||
```
|
||||
|
||||
It must:
|
||||
|
||||
- exact-CAS the current owner;
|
||||
- clear the old record's pane ownership without `onPtyExit`;
|
||||
- leave the old process live and unbound;
|
||||
- move the pane/leaf/handle lookup to the replacement;
|
||||
- invalidate old handles so they cannot silently target the successor; and
|
||||
- correctly reset ID-keyed state when raw PTY ID is reused with a new
|
||||
incarnation.
|
||||
|
||||
Prepare and validate runtime replacement before the Store commit. Make the
|
||||
final in-memory step synchronous and designed not to fail. If an unexpected
|
||||
post-durable failure still occurs, never roll durable ownership back to the old
|
||||
process and never kill either shell; quarantine and retry publication/
|
||||
reconciliation idempotently.
|
||||
|
||||
### 8. Quarantine provider output
|
||||
|
||||
Capture the provider object and generation before spawn and recheck them after
|
||||
every await. Quarantine output for the exact
|
||||
`{ provider generation, ptyId, incarnationId }` until durable and runtime
|
||||
replacement commit.
|
||||
|
||||
Only then activate source delivery and publish the new runtime owner. A new
|
||||
shell reusing the old raw ID must never send bytes into the predecessor model.
|
||||
|
||||
### 9. Fence relay reconnect
|
||||
|
||||
Relay reconnect must acquire the same owner lock after network attach and,
|
||||
before changing ownership or delivery, exact-CAS:
|
||||
|
||||
- the current pane binding;
|
||||
- the lease incarnation; and
|
||||
- the provider/connection generation.
|
||||
|
||||
Only a winning reconnect may set ownership, restore incarnation, activate
|
||||
source delivery, or register runtime state. A reconnect for the predecessor
|
||||
must be unable to publish it after replacement commits.
|
||||
|
||||
### 10. Renderer adoption rules
|
||||
|
||||
The renderer calls only the dedicated API. Guard every completion/failure by:
|
||||
|
||||
- connection object not disposed;
|
||||
- current transport identity;
|
||||
- current action generation; and
|
||||
- matching ticket/operation generation.
|
||||
|
||||
Accept same-raw-ID/new-incarnation as a real successor. Adopt only the exact
|
||||
structured committed result. A stale or disposed callback does nothing: it does
|
||||
not clear a binding, republish the card, or kill a result. Main owns the durable
|
||||
transition, so the renderer must never clear the old durable binding after
|
||||
success.
|
||||
|
||||
## Hard cleanup boundary and selected policy
|
||||
|
||||
Current provider and SSH relay shutdown APIs identify only a PTY ID. If spawn
|
||||
returns `pty-1/inc-B`, then reconnect races and `pty-1` may refer to a different
|
||||
incarnation by the time rollback runs. Calling `shutdown('pty-1')` is not exact
|
||||
cleanup.
|
||||
|
||||
The safe policy for this PR is:
|
||||
|
||||
- do not add replacement intent to the remote wire;
|
||||
- use existing capability-gated create-operation replay and host-attested
|
||||
incarnation when they provide conclusive identity;
|
||||
- clean a failed new shell only when its exact incarnation is provable through
|
||||
a provider-owned rollback handle or equivalent existing evidence;
|
||||
- otherwise park/quarantine the ambiguous shell for reconciliation;
|
||||
- never blind-retry an ambiguous legacy spawn; and
|
||||
- never risk killing the incumbent merely to avoid a leak.
|
||||
|
||||
Legacy relays without create-operation replay or host-attested incarnation must
|
||||
fail closed or leave the outcome unresolved. If implementation cannot satisfy
|
||||
this with current local/provider contracts, stop at the design gate. Do not
|
||||
silently add `expectedIncarnationId` to remote shutdown; that is a separate,
|
||||
capability-negotiated wire design and review.
|
||||
|
||||
## Transaction outcome table
|
||||
|
||||
| Outcome | Durable owner | Process action | Publication |
|
||||
| -------------------------------------------- | ----------------- | ------------------------------------------------- | ------------------------------------- |
|
||||
| Ticket/pre-spawn validation or fence failure | Old | None | Reject |
|
||||
| Spawn failure before dispatch | Old | None | Reject; ticket may be safely terminal |
|
||||
| Spawn response ambiguous after dispatch | Old or unresolved | No blind retry or ID-only shutdown | Park/reconcile |
|
||||
| Known new shell, post-spawn CAS loss | Winner or old | Stop only the exact new incarnation when provable | Publish nothing |
|
||||
| Store commit succeeds | New | Leave old process alive; expire exact old lease | Publish new |
|
||||
| Flush throws; disk has old image | Old | Clean exact new only when provable | Reject |
|
||||
| Flush throws; disk has new image | New | Leave both processes | Publish new |
|
||||
| Flush throws; disk mixed/unreadable | Unresolved | Kill neither | Quarantine/reconcile |
|
||||
| Post-commit runtime/publication failure | New | Never roll back or kill | Retry publication idempotently |
|
||||
|
||||
## Required deterministic oracles
|
||||
|
||||
These are completion requirements, not optional follow-ups.
|
||||
|
||||
### Main transaction
|
||||
|
||||
- Seed an exact failed owner, ticket, binding, and lease.
|
||||
- One explicit replacement call makes exactly one provider fresh spawn.
|
||||
- Provider options contain no session, attach, command, startup, resume,
|
||||
renderer env, launch metadata, or telemetry.
|
||||
- The same pane remains; the old shell receives no shutdown or synthetic exit.
|
||||
- Exactly one combined Store handoff occurs and runtime lookup resolves the new
|
||||
owner.
|
||||
|
||||
### Authorization and lifecycle
|
||||
|
||||
- Missing, expired, replayed, wrong-sender, wrong-host, wrong-worktree,
|
||||
wrong-pane, and renderer-reload tickets reject before provider spawn.
|
||||
- Changed/missing binding, persisted/runtime incarnation, lease, provider, or
|
||||
generation rejects before mutation.
|
||||
- Folder-workspace path changes after ticket issuance are revalidated under the
|
||||
lock.
|
||||
|
||||
### Identity and races
|
||||
|
||||
- Old and new may both be raw `pty-1` with distinct incarnations.
|
||||
- Double-click and lost-response retry create one operation and one shell.
|
||||
- Ordinary spawn racing replacement does not inherit the replacement result.
|
||||
- Reconnect before spawn yields no spawn.
|
||||
- Reconnect after spawn cannot republish the predecessor.
|
||||
- Provider replacement during every await prevents an unsafe commit.
|
||||
- A delayed old-incarnation exit cannot affect the new incarnation.
|
||||
|
||||
### Persistence and durability
|
||||
|
||||
- Success survives save/reload with exactly one flush.
|
||||
- Old exact lease is expired, new exact lease attached, and new binding exact.
|
||||
- Pre-rename failure restores old memory/disk state.
|
||||
- Post-rename failure recognizes the new image as committed.
|
||||
- Mixed/unreadable post-image kills neither process.
|
||||
- Duplicate-retirement rollback cannot revive an already-expired sibling.
|
||||
- Same raw PTY ID with different incarnations remains representable.
|
||||
|
||||
### Runtime and renderer
|
||||
|
||||
- Exact non-exit runtime rebind moves pane/leaf/handle lookup to the successor.
|
||||
- The old owner remains live but unbound and receives no exit.
|
||||
- Disposed, stale-generation, late-success, and late-failure callbacks cannot
|
||||
spawn, clear, kill, or republish over newer state.
|
||||
- Same-ID/new-incarnation structured results are adopted.
|
||||
- A disposed transport never kills a main-committed replacement.
|
||||
|
||||
### Compatibility and E2E
|
||||
|
||||
- No replacement field appears in provider, SSH, relay, runtime RPC, paired
|
||||
RPC, or stream payloads.
|
||||
- Legacy relay degradation is fail-closed/unresolved, never destructive.
|
||||
- Current-to-release and release-to-current terminal compatibility suites stay
|
||||
green.
|
||||
- Unfix the decisive Docker test and add a deterministic production seam that
|
||||
leaves the target connected while one pane attach is unreachable.
|
||||
- The E2E must prove old PID plus start ticks stay alive, exactly one blank
|
||||
shell is added, pane/leaf count is unchanged, new durable `(id,
|
||||
incarnation)` is bound, new lease is attached, old exact lease is expired,
|
||||
and no agent/resume/startup transcript is duplicated.
|
||||
|
||||
## Remaining work, in dependency order
|
||||
|
||||
### Phase 0 — resume safely and freeze the invariants
|
||||
|
||||
1. Read this document and the repository `AGENTS.md` instructions.
|
||||
2. Run the checkpoint commands below.
|
||||
3. Confirm the PR ref remains at the recorded SHA and inspect any newly arrived
|
||||
changes before editing.
|
||||
4. Treat ambiguous cleanup as unresolved rather than adding a remote wire field.
|
||||
5. If that constraint becomes infeasible, stop and return to the user instead
|
||||
of widening scope.
|
||||
|
||||
### Phase 1 — remove the rejected WIP
|
||||
|
||||
Use `apply_patch` to remove only the current six-file
|
||||
`freshSpawnForUnreachablePtyId` and grant-map edits. Preserve every committed
|
||||
change and any unrelated user work. Run `git diff --check` and verify the only
|
||||
remaining change is this handoff document before beginning the replacement.
|
||||
|
||||
### Phase 2 — build the identity and persistence foundation
|
||||
|
||||
1. Add incarnation-aware exact lease identity with legacy-row compatibility.
|
||||
2. Add exact lease lookup/mark/remove APIs required by replacement and
|
||||
reconnect.
|
||||
3. Add the single Store replacement transaction and one-flush rollback state.
|
||||
4. Add post-image inspection for pre-rename, post-rename, and mixed/unreadable
|
||||
outcomes.
|
||||
5. Write persistence tests first and demonstrate the intended failure on the
|
||||
current implementation.
|
||||
|
||||
### Phase 3 — build runtime and coordination primitives
|
||||
|
||||
1. Add the exact non-exit runtime pane-owner replacement primitive.
|
||||
2. Add the host/worktree/pane owner mutex shared by normal spawn,
|
||||
replacement, and reconnect final activation.
|
||||
3. Add operation-token idempotency independent from that lock.
|
||||
4. Update reconnect to revalidate exact binding, lease incarnation, and
|
||||
provider generation after it acquires the lock.
|
||||
5. Add same-ID/new-incarnation and delayed-old-exit tests.
|
||||
|
||||
### Phase 4 — implement the main replacement transaction
|
||||
|
||||
1. Issue typed, sender-bound, expiring, one-shot tickets from every renderer
|
||||
path that can publish the unreachable action.
|
||||
2. Add the dedicated preload/main IPC and main-window sender check.
|
||||
3. Re-derive CWD and validate folder workspaces under the lock.
|
||||
4. Construct the blank provider spawn from the main allowlist.
|
||||
5. Capture provider generation and quarantine exact new output.
|
||||
6. Revalidate after every await; prepare runtime; execute the Store commit;
|
||||
finish runtime activation; publish output/result.
|
||||
7. Implement all transition-table outcomes without ID-only cleanup.
|
||||
|
||||
### Phase 5 — adopt through the renderer
|
||||
|
||||
1. Remove replacement from generic transport/spawn types.
|
||||
2. Thread the typed ticket and structured committed result through the
|
||||
disconnected action only.
|
||||
3. Add disposed/current-transport/action-generation guards.
|
||||
4. Remove renderer-side binding clearing or committed-result killing.
|
||||
5. Verify UI layout, copy, and behavior through Electron Playwright CDP as
|
||||
required by repository instructions.
|
||||
|
||||
### Phase 6 — validate and review once
|
||||
|
||||
1. Run the focused deterministic suite.
|
||||
2. Make the Docker E2E inducible, remove its unconditional `fixme`, and prove
|
||||
it fails for the intended reason on the unfixed candidate before claiming
|
||||
it as evidence.
|
||||
3. Run typecheck, lint, full unit tests, build, reliability gates, and relevant
|
||||
SSH/daemon/cross-version regressions.
|
||||
4. Run independent repository and release-readiness reviews, including
|
||||
security, performance, mobile/backcompat, Windows, Linux/glibc, SSH, folder
|
||||
workspace, and provider-neutrality coverage.
|
||||
5. Resolve every P0/P1 and relevant P2. If another architectural invariant
|
||||
fails, pause instead of entering an unbounded seventh patch loop.
|
||||
|
||||
### Phase 7 — deliver
|
||||
|
||||
Only after the implementation and reviews are clean:
|
||||
|
||||
1. Commit the replacement work.
|
||||
2. Push explicitly to
|
||||
`origin/nwparker/sta-3077-d-affordance-and-review`.
|
||||
3. Update PR #13326's body with the transaction, compatibility behavior, and
|
||||
exact validation receipts.
|
||||
4. Resolve the seven saved review threads only when their findings are actually
|
||||
addressed.
|
||||
5. Request `@coderabbitai review` and monitor CI to completion.
|
||||
6. Use the `orca-linear` workflow to post one concise STA-3077 completion
|
||||
update and attach the PR if needed.
|
||||
|
||||
## First safe resume commands
|
||||
|
||||
Run from the primary worktree before editing:
|
||||
|
||||
```bash
|
||||
git status --short --branch --untracked-files=all
|
||||
git rev-parse HEAD
|
||||
git rev-parse origin/nwparker/sta-3077-d-affordance-and-review
|
||||
git rev-parse origin/main
|
||||
git rev-list --left-right --count origin/main...HEAD
|
||||
git diff --stat
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Expected before any external state changes:
|
||||
|
||||
- HEAD and the PR ref are both `df6edfcebd2ea401a6feda1fccf31189e57eceda`;
|
||||
- the six rejected production-file edits are present;
|
||||
- this handoff file is the only additional local artifact; and
|
||||
- `git diff --check` exits 0.
|
||||
|
||||
Do not use the configured upstream to infer PR state.
|
||||
|
||||
## Suggested validation commands after implementation
|
||||
|
||||
Adjust the focused file list only when the final implementation clearly moves
|
||||
the covered behavior. Record exact commands and outcomes in the PR.
|
||||
|
||||
```bash
|
||||
pnpm exec vitest run --config config/vitest.config.ts \
|
||||
src/main/ipc/pty.test.ts \
|
||||
src/main/persistence.test.ts \
|
||||
src/main/persistence-ssh-lease-termination.test.ts \
|
||||
src/main/runtime/orca-runtime.test.ts \
|
||||
src/main/ssh-reattach-pane-cardinality.test.ts \
|
||||
src/main/ssh/ssh-relay-session.test.ts \
|
||||
src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts \
|
||||
src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts \
|
||||
src/renderer/src/components/terminal-pane/pty-connection.test.ts \
|
||||
src/renderer/src/components/terminal-pane/pty-transport.test.ts \
|
||||
src/renderer/src/components/terminal-pane/TerminalPaneDisconnectedBanner.test.tsx \
|
||||
--reporter=dot
|
||||
|
||||
pnpm typecheck
|
||||
pnpm lint
|
||||
pnpm test
|
||||
pnpm build
|
||||
```
|
||||
|
||||
For the Docker product oracle, use the repo's Electron Playwright configuration
|
||||
with `ORCA_E2E_SSH_DOCKER=1` after adding the deterministic induction seam and
|
||||
removing the test's unconditional `fixme`. Keep the run POSIX-only as the
|
||||
existing Docker fault harness requires.
|
||||
|
||||
Run the current↔release cross-version terminal suite and the relevant Docker
|
||||
SSH cardinality, `MaxSessions=1`, and two-host scripts as part of broad
|
||||
regression coverage.
|
||||
|
||||
## Validation record at pause
|
||||
|
||||
### Current redesign/WIP
|
||||
|
||||
No product validation was run after the rejected six-file WIP was introduced.
|
||||
No product validation was run during the three final read-only audits or while
|
||||
writing this handoff. Therefore this document claims no current test, typecheck,
|
||||
lint, build, Electron, Docker, cross-version, or packaging pass.
|
||||
|
||||
### Read-only checkpoint commands run for this handoff
|
||||
|
||||
The following repository inspections completed successfully:
|
||||
|
||||
```bash
|
||||
git status --short --branch
|
||||
git status --short --branch --untracked-files=all
|
||||
git rev-parse HEAD
|
||||
git branch --show-current
|
||||
git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}'
|
||||
git rev-parse origin/nwparker/sta-3077-d-affordance-and-review
|
||||
git rev-parse origin/nwparker/sta-3077-reattach-pane-cardinality
|
||||
git rev-parse origin/main
|
||||
git rev-list --left-right --count origin/main...HEAD
|
||||
git merge-base origin/main HEAD
|
||||
git log --oneline --decorate -12
|
||||
git reflog --date=iso
|
||||
git diff --stat
|
||||
git diff --name-status
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Observed results are recorded in the repository checkpoint above. In
|
||||
particular, `git diff --check` exited 0.
|
||||
|
||||
Historical validation details are preserved in `goalposts.md` and
|
||||
`resume-plan.md`. Those receipts belong to their recorded SHAs and must not be
|
||||
reported as validation of the final replacement candidate without rerunning
|
||||
them.
|
||||
|
||||
## Security, compatibility, migration, and release risks
|
||||
|
||||
- **IPC authority:** a renderer-supplied PTY ID is forgeable/piggybackable;
|
||||
tickets must be opaque, sender-bound, short-lived, one-shot, and revalidated.
|
||||
- **Same-ID reuse:** raw PTY ID cannot distinguish an incumbent from a
|
||||
successor. Every destructive or ownership operation needs exact incarnation.
|
||||
- **Durability:** a thrown fsync after rename can leave the new image on disk.
|
||||
Blind in-memory rollback can diverge from restart state.
|
||||
- **Remote ambiguity:** ID-only shutdown cannot safely clean same-ID reuse.
|
||||
Prefer a visible/reconcilable leak to killing an unproven process.
|
||||
- **Output isolation:** precommit bytes from a successor can corrupt the old
|
||||
pane model unless quarantined by provider generation plus incarnation.
|
||||
- **Mixed versions:** clients and hosts update independently. Keep replacement
|
||||
local; any future exact-shutdown wire field needs capability negotiation and
|
||||
both-direction live skew tests.
|
||||
- **Legacy persisted rows:** missing incarnation is unknown, not permission to
|
||||
mutate or delete. Migration must be additive and old-reader-safe.
|
||||
- **Folder and SSH workspaces:** re-resolve current paths and host generation
|
||||
under the lock; neither git worktree presence nor local filesystem access may
|
||||
be assumed.
|
||||
- **Cross-platform:** preserve macOS, Linux/glibc floor, Windows, WSL, path, and
|
||||
shortcut rules even though the Docker fault harness is POSIX-only.
|
||||
- **Performance:** avoid global scans, unbounded ticket/operation maps,
|
||||
repeated fsyncs, or hot-path lock contention. Bound retention and measure
|
||||
affected reconnect/spawn paths.
|
||||
- **Release evidence:** the key E2E remains held. The change is not release-
|
||||
ready until that product path is deterministic and discriminating.
|
||||
|
||||
## Stop conditions
|
||||
|
||||
Pause and return to the user if any of these occurs:
|
||||
|
||||
- exact cleanup appears to require an unreviewed remote-wire change;
|
||||
- the Store/runtime transaction cannot distinguish old, new, and mixed durable
|
||||
post-images;
|
||||
- replacement requires treating unknown identity as proof;
|
||||
- the deterministic E2E still cannot enter the actual production state;
|
||||
- the shared lock introduces a new global serialization or deadlock risk; or
|
||||
- the next independent review finds another architectural invariant failure.
|
||||
|
||||
The point of this handoff is to prevent another opaque loop: one bounded
|
||||
redesign, deterministic evidence, one adversarial review, then either delivery
|
||||
or a principled stop.
|
||||
@@ -1,265 +0,0 @@
|
||||
# Terminal session correctness and authority handoff
|
||||
|
||||
> **Status: preserved and paused, not complete, not release-ready.**
|
||||
>
|
||||
> The comprehensive program is at **0/8 proven goalposts** and
|
||||
> **2/13 proven journeys**. No agent may narrow, replace, or mark a goalpost
|
||||
> complete without the user's explicit approval and the proof named in this
|
||||
> folder.
|
||||
|
||||
This folder is the sole continuation entrypoint for the work previously called
|
||||
“React 185” or “terminal session authority.” That shorthand is too narrow: the
|
||||
program covers terminal identity, ownership, exact operations, reconnect,
|
||||
delivery, migration, daemon and SSH lifecycle, paired and remote runtimes,
|
||||
cross-platform compatibility, performance, and removal of superseded code.
|
||||
|
||||
## Read in this order
|
||||
|
||||
1. Read all three preserved source documents with the exact `git show` commands
|
||||
under [Repository checkpoint](#repository-checkpoint). The normative design,
|
||||
delivery ledger, and original pause handoff are mandatory, not optional
|
||||
implementation detail.
|
||||
2. [`goalposts.md`](./goalposts.md) — the current completion contract, amendable
|
||||
only by an explicit user decision.
|
||||
3. [`related-open-work.md`](./related-open-work.md) — live PR/issue overlap and
|
||||
conflicts.
|
||||
4. [`resume-plan.md`](./resume-plan.md) — safe continuation order and required
|
||||
validation receipts.
|
||||
|
||||
The older `terminal-session-authority-handoff-2.md` is useful as an adversarial
|
||||
report, but it is not authoritative. In particular, its “descoped,” “shipped,”
|
||||
identity-equivalence, timer, and LOC claims must not be copied as settled facts.
|
||||
|
||||
## Non-negotiable user objective
|
||||
|
||||
Deliver one coherent long-term terminal-session correctness model that:
|
||||
|
||||
- removes the root causes instead of leaving quarantine, retry-window,
|
||||
sliding-window, optional-metadata repair, or other reconciliation as the
|
||||
final correctness mechanism;
|
||||
- never treats missing, unavailable, timed-out, disconnected, or unknown state
|
||||
as proof that a process is dead or that replacement/destruction is safe;
|
||||
- works for local terminals, daemon, WSL, direct and nested SSH, paired runtime,
|
||||
remote server, folder workspaces, floating workspaces, and git worktrees;
|
||||
- remains correct when clients and hosts update independently;
|
||||
- works on macOS, Linux at the supported glibc floor, Windows, WSL, and Docker
|
||||
OpenSSH;
|
||||
- has no correctness, security, latency, throughput, memory, restore, startup,
|
||||
backpressure, or large-pane regression;
|
||||
- finishes with **strictly less aggregate production code than the frozen
|
||||
pre-program baseline** after all superseded implementations are deleted; and
|
||||
- is proven through real production paths and discriminating end-to-end
|
||||
journeys, not inferred from test counts or green CI.
|
||||
|
||||
Correctness and performance cannot be weakened to reach the code-size target.
|
||||
The code-size target also cannot be waived because a replacement architecture
|
||||
was added beside the old one.
|
||||
|
||||
## Authority and scope rules
|
||||
|
||||
- Direct user instructions govern scope.
|
||||
- The original normative design remains the contract except where a later,
|
||||
explicit user decision changes a named requirement.
|
||||
- This handoff tightens G6 to **strictly net-negative production LOC** because
|
||||
the user explicitly asked to end with less production code than the original
|
||||
baseline.
|
||||
- The default frozen accounting baseline is
|
||||
`5ed45739e94bdf6460364e033bfcec9b32c0b42a`, the base recorded by GitHub for
|
||||
PR #12600. This program subsumes that containment PR. Every later
|
||||
program-attributable merged prerequisite and every stack layer must be counted
|
||||
in aggregate; rebasing may not roll the accounting baseline forward and hide
|
||||
earlier additions. Changing this baseline requires an explicit user decision.
|
||||
- Focused tests, green CI, a review comment, a behavior-contract document, an
|
||||
open PR, or a construction milestone cannot amend the design or promote a
|
||||
release goalpost.
|
||||
- A replacement design is allowed, but it must map every original invariant,
|
||||
risk, and journey to an equal or stronger mechanism, close the known liveness
|
||||
gap, and receive explicit user approval before the ledger changes.
|
||||
- Only `not started`, `partial`, and `proven` are valid gate statuses.
|
||||
“Descoped,” “superseded,” and “met for what shipped” are not valid statuses
|
||||
for G0–G7 under the current objective.
|
||||
- A narrow incident PR may be independently useful or mergeable. It does not
|
||||
reduce the comprehensive denominator of eight goalposts and thirteen
|
||||
journeys.
|
||||
|
||||
## Settled status
|
||||
|
||||
| Scope | Status | Meaning |
|
||||
| -------------------------- | --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Construction preservation | **Complete locally; not remotely durable** | The 814-path construction tree and the original three authority documents are preserved at local branch `nwparker/react185-authority-snapshot` / `fddb19f6977`. |
|
||||
| Comprehensive goalposts | **0/8 proven** | G0–G4 and G6 are partial. G5 and G7 are not started. |
|
||||
| Required journeys | **2/13 proven** | Journeys 1 and 2 have clause-selective discriminating oracles run on every environment they name. The other eleven remain unproven. |
|
||||
| Current branch / PR #13111 | **Implemented, open, unapproved, incomplete** | Useful narrow SSH containment; not a long-term completion and not merged. |
|
||||
| PR #13110 | **Open, independent, unapproved** | Useful preload artifact guard; not shipped and has an unresolved major review thread. |
|
||||
| Less-code gate | **Failed on current candidate** | PR #13111 is net positive; the preserved construction snapshot is net `+60,903` production LOC. |
|
||||
|
||||
The numeric release checkpoint is therefore:
|
||||
|
||||
- **G0:** partial
|
||||
- **G1:** partial
|
||||
- **G2:** partial
|
||||
- **G3:** partial
|
||||
- **G4:** partial
|
||||
- **G5:** not started
|
||||
- **G6:** partial and currently failing its final LOC condition
|
||||
- **G7:** not started
|
||||
- **Total:** **0/8 proven, 6/8 partial, 2/8 not started**
|
||||
- **Journeys:** **2/13 proven** (Journey 1 natively on macOS/Linux/Windows; Journey 2 on macOS/Linux/physical WSL)
|
||||
|
||||
This is the **program artifact ledger**. Its partial G0, G1, G3, and G4 evidence
|
||||
exists only in the preserved construction snapshot and historical receipts. It
|
||||
does not mean those implementations exist on the current #13111 branch. The
|
||||
current candidate contributes credible partial G2 evidence and limited
|
||||
compatibility/review evidence, but it proves **zero** complete release gates.
|
||||
|
||||
## Repository checkpoint
|
||||
|
||||
Verified before creating this documentation folder:
|
||||
|
||||
- Worktree:
|
||||
`/Users/nwparker/orca/workspaces/orca/eye-React-185`
|
||||
- Current branch: `nwparker/sta-3077-reattach-pane-cardinality`
|
||||
- Current HEAD: `5369479be2953f45cea9ab5cfcde756dd9660548`
|
||||
- HEAD subject: `fix(pty): let liveness say unknown instead of forcing it to say dead`
|
||||
- Local `origin/main`: `0d29497f8279e2d4c2d26ffc8f3fb68cee2756a7`
|
||||
- Merge base: `e6e197feeddd1adda066c5cd22f7ee056a12c8c1`
|
||||
- Distance from local `origin/main`: 15 commits ahead, 19 behind
|
||||
- The worktree was clean before this handoff. The expected handoff-only changes
|
||||
are this folder and its `.gitignore` allow-list entry.
|
||||
- Preserved construction branch:
|
||||
`nwparker/react185-authority-snapshot`
|
||||
- Preserved construction commit:
|
||||
`fddb19f6977ae4ba4764c32afc9fe104c1ed2549`
|
||||
- Preservation location: local ref only; the branch has no upstream, tag, or
|
||||
remote branch containing this commit
|
||||
- Snapshot status: **unshippable salvage/reference only**
|
||||
|
||||
The exact commit ID is the preservation anchor. Do not checkout, switch, reset,
|
||||
rebase, commit, or push from the snapshot branch in the primary worktree. Read
|
||||
its authority documents without switching branches:
|
||||
|
||||
```bash
|
||||
git show fddb19f6977:docs/reference/terminal-session-authority.md
|
||||
git show fddb19f6977:docs/reference/terminal-session-authority-delivery.md
|
||||
git show fddb19f6977:docs/reference/terminal-session-authority-handoff.md
|
||||
```
|
||||
|
||||
Mine tests, design arguments, or narrow implementations only through `git show`
|
||||
or a disposable isolated worktree, and only after verifying that they address a
|
||||
reachable production path on the current base. A local tag protects against
|
||||
branch movement and ordinary GC only; it does not survive loss of this machine.
|
||||
Cross-machine/crash preservation requires a copy-verified off-machine bundle or
|
||||
an explicitly authorized remote ref. Do not infer permission to push it.
|
||||
|
||||
## Current narrow candidate: what is real
|
||||
|
||||
At current HEAD / PR
|
||||
[#13111](https://github.com/stablyai/orca/pull/13111):
|
||||
|
||||
- SSH relay reattach passes `mayCreate: false` to stop that call site from
|
||||
creating durable pane/layout state.
|
||||
- Duplicate complete pane leases are superseded and excluded from reattach.
|
||||
- Existing duplicate complete leases are reconciled during reconnect.
|
||||
- Reattach failure paths require positive evidence before cold respawn.
|
||||
- PTY inventory can report `unknown` rather than collapsing an unavailable
|
||||
provider into `false`/dead.
|
||||
- Focused tests and GitHub checks are green at HEAD.
|
||||
|
||||
These are credible partial G2 improvements. They do **not** establish the
|
||||
comprehensive design or release gates.
|
||||
|
||||
## Current narrow candidate: what remains unproved or wrong
|
||||
|
||||
- Lease row identity is still `(targetId, ptyId)`. Pane fields remain optional.
|
||||
Active-pane cardinality is enforced by sibling expiration and a healing scan,
|
||||
so the end state remains reconciliation-based rather than structurally
|
||||
pane-keyed.
|
||||
- Incomplete legacy leases bypass pane arbitration.
|
||||
- Superseded remote shells are deliberately left running and unreachable. The
|
||||
PR body acknowledges that the “accumulates unused shells” half of the incident
|
||||
remains unresolved.
|
||||
- The repeated-reconnect Docker test still passes with and without the fix: it
|
||||
is a forward guard, not causal proof. Its sibling in the same spec — 'leaves a
|
||||
lease whose durable pane is gone unbound…' — was reported as discriminating,
|
||||
but that DID NOT REPRODUCE on a second machine: with `mayCreate: false`
|
||||
removed from the reattach call site and the app rebuilt, both tests still
|
||||
passed. Its induction races `pty:kill` against a severed transport, so when
|
||||
the kill lands the lease is cleaned up and there is nothing to graft. Treat
|
||||
both tests as forward guards. No journey is proven.
|
||||
- The final Docker settle assertion has an unresolved major review thread: a
|
||||
late pane or shell can appear after the polling assertion has already passed.
|
||||
- The production-call-site wiring oracle reads source text. It does not execute
|
||||
the full production reattach path.
|
||||
- `mayCreate: false` is wired at one production binding call site, not proven
|
||||
across every recovery/reattach runtime. `removeSshRemotePtyLease` still has one
|
||||
production caller, the spawn-persistence rollback.
|
||||
- `WEDGED_DAEMON_GRACE_RETRIES = 11` remains reachable and still permits retry
|
||||
count to drive daemon replacement after unknown inventory.
|
||||
- `terminal-input-quarantine.ts` remains reachable.
|
||||
- No current-tree proof covers physical Windows, WSL, the Linux glibc floor,
|
||||
paired runtime, remote server, two independent SSH hosts, live mixed-version
|
||||
peers, or production-scale performance.
|
||||
- There is no candidate-versus-baseline performance result for the new
|
||||
per-upsert sibling scan or reconnect healing pass.
|
||||
- PR #13111 is open, has no approval decision, and is not shipped.
|
||||
|
||||
Current HEAD census against its merge base, before this documentation change:
|
||||
|
||||
| Category | Additions | Deletions | Net |
|
||||
| --------------------------------------------------- | --------: | --------: | -------: |
|
||||
| Production source, including a fixture under `src/` | 374 | 65 | **+309** |
|
||||
| Tests and end-to-end tests | 944 | 30 | +914 |
|
||||
| Documentation | 146 | 0 | +146 |
|
||||
| CI/configuration | 43 | 0 | +43 |
|
||||
| Total GitHub diff | 1,507 | 95 | +1,412 |
|
||||
|
||||
The production source plus Docker runner is net `+350`. Whichever taxonomy is
|
||||
used, the categories must remain separate and G6 is not proven.
|
||||
|
||||
## Corrections to the previous handoff
|
||||
|
||||
- PRs #13110 and #13111 are **open and green**, not shipped.
|
||||
- #13110 is implementation-independent from #13111, but emitted-preload safety
|
||||
remains inside G0/G5/G7 and any retained #13110 production/build logic counts
|
||||
in the aggregate G6 census.
|
||||
- The snapshot genuinely failed to wire `mayCreate`, pane-cardinality
|
||||
arbitration, or a complete shell-lifecycle fix into the affected legacy SSH
|
||||
path.
|
||||
- Pane key, PTY ID, incarnation ID, a narrow CAS, and a three-valued daemon
|
||||
resolver pre-existed and should be reused where their semantics match.
|
||||
- Existing `connectionId` and `worktreeId` do **not** establish equivalence to a
|
||||
final-host-minted authority identity and canonical host-local namespace.
|
||||
- The normative design never banned all timers. It permits bounded waits and
|
||||
backoff; it forbids elapsed time or retry count **alone** from deciding
|
||||
identity, liveness, takeover, replacement, or destruction.
|
||||
- The design does have a liveness gap: a permanently lost retained consumer can
|
||||
hold compaction at capacity without a sufficiently explicit bounded operator
|
||||
recovery path. G0 and G3 cannot become proven until that is resolved.
|
||||
- The claimed approximately 6,500-line comparison implementation has no
|
||||
reproducible census and cannot justify deleting requirements.
|
||||
|
||||
## Definition of done
|
||||
|
||||
The comprehensive work is done only when all of the following are true on one
|
||||
rebased, converged candidate SHA:
|
||||
|
||||
- every G0–G7 section in [`goalposts.md`](./goalposts.md) has its named
|
||||
production implementation and current proof;
|
||||
- all thirteen required journeys pass with their full oracles;
|
||||
- each new causal oracle is red on the unfixed baseline and green on the final
|
||||
candidate; formal verification may supplement but never replace a required
|
||||
live journey;
|
||||
- every open incident bound to the mandatory issue-to-journey matrix in
|
||||
[`related-open-work.md`](./related-open-work.md) passes its discriminating
|
||||
oracle or is ruled unrelated with explicit evidence and user acceptance;
|
||||
- correctness, security, wire, migration, platform, packaging, performance,
|
||||
restore, and scale reviews have no unresolved P0–P2 findings;
|
||||
- the aggregate production source census is strictly net-negative against the
|
||||
frozen pre-program baseline, including all program-attributable prerequisite
|
||||
PRs and stack layers;
|
||||
- no superseded reconciliation, quarantine, duplicate state machine, legacy
|
||||
writer, or test fixture remains reachable in production; and
|
||||
- the user explicitly accepts any deliberate change to the normative design.
|
||||
|
||||
Anything less must be reported as partial, regardless of test count or PR
|
||||
state.
|
||||
@@ -1,111 +0,0 @@
|
||||
# Resume prompt — terminal session ownership
|
||||
|
||||
Paste the block below into a fresh session. Everything it needs is in the repo.
|
||||
|
||||
---
|
||||
|
||||
You are taking over an in-flight implementation in this repo. Read these, in
|
||||
order, before doing anything:
|
||||
|
||||
1. `docs/reference/terminal-session-correctness-handoff-2026-08-07/new-design-goalposts.md`
|
||||
— the tracked goalposts. This is the contract. Report progress against it.
|
||||
2. `docs/reference/terminal-session-correctness-handoff-2026-08-07/design-explained.html`
|
||||
— the design in plain English (open it; it has diagrams).
|
||||
3. `docs/reference/terminal-session-correctness-handoff-2026-08-07/design-final-detail.md`
|
||||
— the full detail: record shape, death rule, reattach algorithm, migration
|
||||
order, and the oracle list.
|
||||
4. `docs/reference/terminal-session-correctness-handoff-2026-08-07/counsel-design.html`
|
||||
— how the design was reviewed, and the defects the review found in earlier
|
||||
shipped work. Read at least the "What I got wrong" section.
|
||||
|
||||
## The problem being fixed
|
||||
|
||||
Reconnecting to an SSH host multiplied a user's terminals (2 → 19 → 20), and a
|
||||
coding agent could be resumed twice into one transcript. Root cause is not one
|
||||
bug: it is that ownership bookkeeping is filed under the wrong key and written to
|
||||
two places, and that the client infers "the program died" from things that do not
|
||||
prove it.
|
||||
|
||||
The design in one line: **keep our own notes about which terminal is whose, and
|
||||
never guess whether a program died.**
|
||||
|
||||
## State
|
||||
|
||||
Branch `nwparker/sta-3077-reattach-pane-cardinality`. Landed and committed:
|
||||
|
||||
- `e2524b0472f` — an identity mismatch is no longer read as death (goalpost S1)
|
||||
- `c51be8072ba` — pane identity is no longer sent on reattach (S2)
|
||||
- Goalposts, design docs (S6 verdict recorded)
|
||||
|
||||
**3 of 7 step goalposts proven. Net production lines: −24.**
|
||||
|
||||
Two older PRs (#13110, #13111) are **held, unmerged, deliberately**. The owner
|
||||
decided: no stopgap, get the design right. Do not merge them without being asked.
|
||||
|
||||
## What to do next
|
||||
|
||||
Three goalposts have **failing oracles already written**, each with the exact
|
||||
production change identified. This is implementation against a fixed target.
|
||||
|
||||
**S3 — a failed reattach must never fabricate an exit.**
|
||||
Oracle: `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts` (6 red, 2 green).
|
||||
In `handlePtyReattachFailure` (`src/main/ssh/ssh-relay-session.ts`), the
|
||||
not-found case currently sends the pane a synthetic `pty:exit { code: -1 }`,
|
||||
clears provider state, deletes ownership and expires the lease — all claims about
|
||||
a process we know nothing about. Collapse it into the non-destructive branch that
|
||||
already exists a few lines above (`restoreRequired = 'reattachAttemptsExhausted'`
|
||||
+ `wakeRecovery`). This is a branch collapse, not a new mechanism.
|
||||
Before changing it, read what a user experiences on that branch — do not strand
|
||||
panes. This step ships with the disconnected-pane affordance below.
|
||||
|
||||
**S4 — one partition per (target, pane).** *Highest value: this is the 2 → 19 → 20
|
||||
mechanism.* Oracle: `src/main/ssh-pane-binding-partition.test.ts` (6 red).
|
||||
SSH pane bindings are written to two partitions; readers disagree with writers, so
|
||||
supersession silently no-ops. Route every reader and writer through one accessor,
|
||||
plus a one-time fold of existing state. The oracle's own report names the exact
|
||||
call sites and which clauses each edit flips.
|
||||
|
||||
**S5 — the superseded-pane fence must be live on the reattach path.**
|
||||
Oracle: `src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts` (2 red).
|
||||
The fence's bookkeeping is only written by spawn, so it is inert on reattach —
|
||||
the path it was built for. Collapse to one `bindPaneShell` producer used by all
|
||||
paths.
|
||||
|
||||
**S8 — remove the dead recovery-grant path.** Proven unreachable (see S6 in the
|
||||
goalposts). Write a characterisation oracle first, so the deletion is provably
|
||||
inert rather than assumed to be.
|
||||
|
||||
**Product affordance (ships with S3).** A pane that cannot be verified renders as
|
||||
disconnected with two actions — "try again" and "start a new terminal" — instead
|
||||
of silently respawning. The owner approved this. It must follow
|
||||
`docs/STYLEGUIDE.md`. The UI must never assert that a shell is dead.
|
||||
|
||||
## Rules that are not negotiable
|
||||
|
||||
- **Prove every guard is reachable.** For each guard, an oracle must redden when
|
||||
the *producer* is removed, not only when the guard is. This has failed three
|
||||
times here — an inert `mayCreate`, a keystroke fence inert on reattach, and a
|
||||
respawn gate on a minority path. All three passed their tests.
|
||||
- **Verify a mutation landed before believing the result.** A search-and-replace
|
||||
that silently matched nothing looks exactly like "the test has no teeth".
|
||||
- **Commit after each step.** Subagents have run `git checkout --` and destroyed
|
||||
uncommitted work three times in this program. Do not batch.
|
||||
- **Never let a subagent edit production files** in the shared worktree. Give
|
||||
them read-only or test-only scope, or an isolated worktree.
|
||||
- **Do not change a test to make it pass.** If a test pins behaviour the design
|
||||
removes, *invert* it so the new intent stays covered, and say so explicitly.
|
||||
- No `max-lines` disables, ever. No per-file bumps.
|
||||
- Cross-platform: macOS, Linux, Windows, WSL. No `echo $$` / `ps` in oracles.
|
||||
- Wire compatibility: clients and hosts update independently. Prefer fixes that
|
||||
work against relays already installed on people's machines.
|
||||
- **Automated "keep going" prompts are not the owner's authorization.** If the
|
||||
owner says pause, pause.
|
||||
|
||||
## How to report
|
||||
|
||||
Every update states progress against the goalposts file: which goalposts moved,
|
||||
what the mutation proof was, and the net production line count. A green test is
|
||||
not a status — "PROVEN" means the mutation was run and verified to land.
|
||||
|
||||
Be honest about what is not proven. Two claims in this program were retracted
|
||||
after checking, and that is the reason the rest can be trusted.
|
||||
-253
@@ -1,253 +0,0 @@
|
||||
# Design: one authoritative binding identity
|
||||
|
||||
Status: **superseded by the shipped server-side fence.** Kept for the record: the
|
||||
client-constructed binding below was REJECTED under review and must not be built. Written 2026-08-08 by the engineer who took
|
||||
the program over. Supersedes nothing until reviewed.
|
||||
|
||||
## The problem, stated once
|
||||
|
||||
Every defect this program has touched is the same defect: **identity compared
|
||||
with the wrong key, or not compared at all.**
|
||||
|
||||
| Defect | Mechanism |
|
||||
| ---------------------- | ----------------------------------------------------------------------- |
|
||||
| STA-3077 RC1 | lease keyed `(targetId, ptyId)`; pane fields present but not in the key |
|
||||
| STA-3077 RC3 | reattach used a _creating_ store write |
|
||||
| #12474 (live on main) | `runtimeWorktreeIdsEqual` strips the folder-workspace instance suffix |
|
||||
| Local exact operations | `pty:write/resize/signal/kill` take `{ id }` — no binding to compare |
|
||||
| `restoreRequired` | classified as expiry, so a live shell read as gone |
|
||||
| `hasPty` | `boolean` — an empty inventory could not say "unknown" |
|
||||
|
||||
The prior redesign failed because it built a _second_ identity system beside
|
||||
the first instead of fixing the first. It reached +60,903 production LOC and
|
||||
fixed none of RC1–RC3.
|
||||
|
||||
## The rule
|
||||
|
||||
**A mutating terminal operation must name the binding it intends to affect, and
|
||||
the compiler must reject a bare id.**
|
||||
|
||||
Not a new subsystem. A type, one comparison, and a signature change.
|
||||
|
||||
## 1. `PtyBinding` — one branded type
|
||||
|
||||
```ts
|
||||
declare const bindingBrand: unique symbol
|
||||
|
||||
export type PtyBinding = Readonly<{
|
||||
hostId: ExecutionHostId // exists — LOCAL_EXECUTION_HOST_ID | ssh:<target>
|
||||
worktreeId: WorktreeId // exists — `${repoId}::${path}[::workspace:<uuid>]`
|
||||
paneKey: PaneKey // exists — `${tabId}:${leafId}`, already branded
|
||||
ptyId: string // exists
|
||||
incarnationId: PtyIncarnationId // exists — already branded
|
||||
}> & { readonly [bindingBrand]: true }
|
||||
```
|
||||
|
||||
Every field already exists and is already persisted. Nothing is invented.
|
||||
|
||||
Construction is the whole point: `PtyBinding` is producible **only** by
|
||||
`bindingFromAuthority()` — reading the durable store, a spawn result, or an
|
||||
attach reply. There is no public constructor from loose strings, so a caller
|
||||
cannot fabricate one, and `as PtyBinding` is banned by lint.
|
||||
|
||||
## 2. Mutating IPC carries the binding
|
||||
|
||||
Today, on the local path:
|
||||
|
||||
```ts
|
||||
pty: write({ id, data })
|
||||
pty: resize({ id, cols, rows })
|
||||
pty: signal({ id, signal })
|
||||
pty: kill({ id })
|
||||
```
|
||||
|
||||
There is no fence to test because there is nothing to compare. That is a
|
||||
production gap, not a test gap.
|
||||
|
||||
After:
|
||||
|
||||
```ts
|
||||
pty: write({ binding, data })
|
||||
pty: resize({ binding, cols, rows })
|
||||
pty: signal({ binding, signal })
|
||||
pty: kill({ binding })
|
||||
```
|
||||
|
||||
The handler resolves the binding against the authoritative record and rejects a
|
||||
mismatch — the same compare-and-swap `persistPtyBinding` already performs for
|
||||
`expectedBinding`. A stale renderer cannot reach a successor pane or a reused
|
||||
PTY id, which is invariants 1–4 of the original design, enforced rather than
|
||||
asserted.
|
||||
|
||||
**Compatibility:** the ID-only channels stay for one release behind the existing
|
||||
capability negotiation, since clients and hosts update independently. They are
|
||||
marked deprecated, are not reachable from authoritative paths, and are deleted
|
||||
in the release after — that deletion is where the LOC comes back.
|
||||
|
||||
## 3. One comparison, not twenty-six
|
||||
|
||||
```ts
|
||||
export function bindingsEqual(a: PtyBinding, b: PtyBinding): boolean
|
||||
export function sameNamespace(a: PtyBinding, b: PtyBinding): boolean
|
||||
```
|
||||
|
||||
Delete the hand-rolled comparisons. `runtimeWorktreeIdsEqual` — the #12474 bug —
|
||||
is one of them; ~26 repeat a host-id/namespace-id comparison inline. Each
|
||||
hand-rolled copy is a future drift, and #12474 proves drift already happened.
|
||||
|
||||
## 4. Three-valued liveness
|
||||
|
||||
Landed (`5369479be29`). `hasPty: boolean | null`, `null` never authorizes
|
||||
destruction.
|
||||
|
||||
## What this deletes
|
||||
|
||||
| Target | Est. LOC |
|
||||
| ----------------------------------------------------------------- | -------: |
|
||||
| ID-only IPC channels + handlers (release after next) | ~250 |
|
||||
| Hand-rolled identity comparisons (~26 sites) | ~180 |
|
||||
| `pty-source-replay-index.ts` (done) | 201 |
|
||||
| Inference sites that exist only because a binding was unavailable | ~200 |
|
||||
|
||||
Net direction is negative once the deprecated channels go. It is net-positive in
|
||||
the release that adds the type, and the recorded G6 decision permits that when
|
||||
justified.
|
||||
|
||||
## What this deliberately does not build
|
||||
|
||||
No durable journal, no per-consumer cumulative cursors, no cryptographic device
|
||||
principals, no parallel authority service. Verified against mature prior art:
|
||||
bounded replay plus lifecycle-in-the-attach-reply is what shipping systems use,
|
||||
and their entire persistent-terminal subsystem is ~6,500 LOC.
|
||||
|
||||
If a reviewer can name a concrete event sequence where bounded replay loses or
|
||||
double-applies an outcome, that conclusion changes. Nobody has yet.
|
||||
|
||||
## How each claim gets falsified
|
||||
|
||||
| Claim | Oracle |
|
||||
| ---------------------------------------- | ---------------------------------------------------------------------- |
|
||||
| A stale op cannot reach a successor pane | Send a captured binding after the pane is recreated; must be refused |
|
||||
| A stale op cannot reach a reused pty id | Same, after incarnation change |
|
||||
| Same-path workspaces do not collide | The 5 skipped tests in `workspace-namespace-terminal-identity.test.ts` |
|
||||
| Reattach never creates | Proven, discriminating (`ed10a467883`) |
|
||||
| Unknown never destroys | Revert three-valued `hasPty`; daemon spec reddens |
|
||||
|
||||
Every row must fail with its guard removed, verified under an isolated
|
||||
`TMPDIR` — the e2e harness keys its seeded-repo pointer on a machine-global
|
||||
tmpdir path, so a shared machine can both fabricate and mask a red.
|
||||
|
||||
---
|
||||
|
||||
# Post-review record (2026-08-08)
|
||||
|
||||
## The proposal above was rejected, correctly
|
||||
|
||||
Verified against the code, three load-bearing claims in it were false: there is
|
||||
no `WorktreeId` type; `PtyIncarnationId` is a bare `string` alias, not branded;
|
||||
and `incarnationId` is optional in the store and never reaches the renderer, so
|
||||
the side asked to construct the binding cannot. It was also a _second_ identity
|
||||
comparison beside `resolveStablePaneOwner` (`src/main/ipc/pty.ts:647`), which
|
||||
already resolves pane ownership against runtime and store and already throws
|
||||
`terminal_pane_owner_conflict` — with 9 call sites, all on spawn/adopt, none on
|
||||
a mutating handler. The fence existed; it was not being called.
|
||||
|
||||
## What shipped instead
|
||||
|
||||
`isSupersededPtyId` in `src/main/ipc/pty.ts`, consulted by `pty:write`,
|
||||
`pty:writeAccepted`, `pty:resize` and `pty:signal`. Main keeps `ptyPaneKey` and
|
||||
`paneKeyPtyId` in lock-step, so their disagreement is proof the caller's id was
|
||||
superseded. No wire change, no renderer change, nothing added to the input
|
||||
payload. `pty:kill` is deliberately exempt: a superseded PTY is orphaned and
|
||||
reclaiming it is the point.
|
||||
|
||||
## Peer review of that shape
|
||||
|
||||
Four comparable agent IDEs were studied. The verdict was to keep this shape.
|
||||
None sends a composite identity per keystroke; the one that fences keystrokes
|
||||
sends a single opaque server-minted scalar. The best peer formulation is _keep
|
||||
the id as the sole lookup key and make the second field a rejection predicate,
|
||||
never a key component_ — which is what this does, with the owner read from
|
||||
authority rather than accepted from the caller.
|
||||
|
||||
## Known gap, stated precisely
|
||||
|
||||
**The fence compares a binding, not an incarnation.** If a pane respawns under
|
||||
the _same_ ptyId, `paneKeyPtyId.get(paneKey) === ptyId` still holds and a write
|
||||
composed for the dead shell passes.
|
||||
|
||||
The obvious remedy — mint a fresh ptyId on every spawn — is wrong here.
|
||||
`ptySessionIdForAgentCreateOperation` (`src/main/daemon/pty-session-id.ts:26`)
|
||||
is deterministic _by design_ so a replayed agent-create is idempotent rather
|
||||
than spawning a second shell. Randomising it would trade this narrow gap for a
|
||||
duplicate-spawn bug, which is worse.
|
||||
|
||||
So the residual exposure is one path: an agent-create PTY dies, the same
|
||||
operation is replayed, the id is reproduced, and a caller holding a pre-death
|
||||
reference writes into the successor. Closing it needs the caller to carry the
|
||||
incarnation — the rejected proposal — or a per-attachment token, which is a wire
|
||||
change. Neither is justified by evidence today. Recorded rather than hidden.
|
||||
|
||||
## Adopted from the peer review
|
||||
|
||||
- The "no binding recorded" branch is an explicit, commented decision (permit),
|
||||
not an accident — an omitted guard on an optional field is exactly how one
|
||||
peer reintroduced this defect class.
|
||||
- A classifier's default must be non-destructive: unknown resolves to reattach,
|
||||
and only a positive "session not found" authorises a respawn.
|
||||
|
||||
## Not yet adopted, ranked
|
||||
|
||||
1. Typed end-reason recorded at end time (`detached` vs `terminal-exited`), so a
|
||||
user quit is not a resume candidate. Aimed squarely at duplicate resume.
|
||||
2. Compare-and-swap before any _delayed_ destructive write. Three-valued probes
|
||||
stop us acting on unknown; CAS stops us acting on stale-known.
|
||||
3. Derive the pane inventory from the authority snapshot rather than merging into
|
||||
a local store. A projection cannot grow; an accumulator can — which is what
|
||||
2 -> 19 -> 20 was.
|
||||
4. Durable intent-to-kill, since kill is the reclamation path for orphaned leases
|
||||
and a one-shot renderer broadcast should not be its only chance.
|
||||
|
||||
---
|
||||
|
||||
# Duplicate agent resume: why I did not add the fix I recommended
|
||||
|
||||
Adoption item 1 was a typed end-reason recorded at end time — `detached` (the
|
||||
user quit; not a resume candidate) versus `terminal-exited` (it died under the
|
||||
agent; resumable) — on the grounds that a boolean `ended` cannot express the
|
||||
difference, which is why systems keep resuming things they should not.
|
||||
|
||||
I went to implement it and stopped, because the codebase already contains that
|
||||
idea three times over.
|
||||
|
||||
`SleepingAgentSessionRecord` (`src/shared/agent-session-resume.ts`) carries:
|
||||
|
||||
- `origin?: 'worktree-sleep' | 'quit' | 'live'` — added so worktree activation
|
||||
would not launch a tab that a warm reattach had already restored (#5232);
|
||||
- `restoreOnTabOpenOnly?: boolean` — added so a mobile wake would not background
|
||||
mount every slept tab and respawn the workspace the user just slept (#11598);
|
||||
- `automaticResumeBlockedBy?: 'legacy-orchestration-worker'` — added so a
|
||||
relaunch could not race a durable orchestration assignment.
|
||||
|
||||
Three fields, three incidents, one defect: something resumed that should not
|
||||
have. They are consulted at 22 non-test sites. A fourth flag — however well
|
||||
typed — is the fifth containment cycle, which is the failure pattern this
|
||||
program exists to break.
|
||||
|
||||
The peer designs that do not have this bug do not have a better flag. They have
|
||||
a different shape:
|
||||
|
||||
1. **Nothing resumes automatically.** Resume is a button. On click it resumes
|
||||
into a _brand-new terminal id_, re-points the pane, then kills the old
|
||||
terminal — so two live agents for one pane never exist even transiently.
|
||||
2. **Two-agents-in-one-terminal is unrepresentable.** The terminal id is the
|
||||
agent binding's primary key with a cascade to the session row, and live reads
|
||||
inner-join on the session being active — so a dead terminal's agent cannot
|
||||
appear in a read no matter how the terminal died.
|
||||
|
||||
Neither is a 30-line change here, and (1) is a product decision about whether
|
||||
automatic resume stays a feature. That belongs to the user, not to me.
|
||||
|
||||
What is safe to say: the next duplicate-resume incident should not be answered
|
||||
with a fourth predicate on this record. The record already proves that approach
|
||||
does not converge.
|
||||
File diff suppressed because one or more lines are too long
-186
@@ -1,186 +0,0 @@
|
||||
# Design brief — terminal session ownership & transport
|
||||
|
||||
Review artifact. Repo: Orca (Electron + renderer + relay). Everything below marked
|
||||
**[V]** was verified by reading the code at the cited location. Everything marked
|
||||
**[R]** is agent-reported and only spot-checked. Treat **[R]** as a lead, not a fact.
|
||||
|
||||
---
|
||||
|
||||
## 1. The product problem
|
||||
|
||||
Orca runs terminals on four "routes":
|
||||
|
||||
| Route | Provider | Survives app restart |
|
||||
| --------------------- | ----------------------------------------------- | -------------------------- |
|
||||
| local in-process | `LocalPtyProvider` | no |
|
||||
| local daemon | `DaemonPtyAdapter` (swapped into the same slot) | yes |
|
||||
| direct SSH | `SshPtyProvider` → remote relay | yes, within a grace window |
|
||||
| paired remote runtime | not an `IPtyProvider`; runtime RPC | yes |
|
||||
|
||||
Reported failures:
|
||||
|
||||
1. **Pane cardinality growth (STA-3077).** SSH reconnect: 2 terminals → 19 → 20.
|
||||
Most are panes the user never opened. Remote host fills with unused shells.
|
||||
2. **Duplicate agent resume.** A coding agent gets resumed twice, two processes
|
||||
writing one conversation file. One report reached five.
|
||||
3. **Blast radius.** One PTY failing to re-prove its output stream could drop the
|
||||
whole host connection, killing every pane, file transfer and git command on it.
|
||||
|
||||
## 2. Verified facts (the evidence base)
|
||||
|
||||
**[V] Dispatch is a substitution for daemon, a parallel registry for SSH.**
|
||||
`src/main/ipc/pty.ts:932-941`:
|
||||
|
||||
```ts
|
||||
function getProvider(connectionId) {
|
||||
if (!connectionId) return localProvider // local AND daemon — one slot
|
||||
const provider = sshProviders.get(connectionId) // SSH — separate map
|
||||
}
|
||||
```
|
||||
|
||||
The daemon is installed by `setLocalPtyProvider(routedAdapter)`
|
||||
(`src/main/daemon/daemon-init.ts:1031`) and adds **zero** branches to the shared
|
||||
spine. The paired runtime is encapsulated behind a `PtyTransport`. SSH is neither —
|
||||
it rides the local path carrying a `connectionId` tag.
|
||||
|
||||
**[R] That tag leaks:** ~199 remoteness branches in main (52 in `pty.ts`, 143 in
|
||||
`orca-runtime.ts`), ~191 in the renderer, plus 25 `instanceof` provider checks.
|
||||
The `remote:` predicate is re-declared 5 times in the renderer.
|
||||
|
||||
**[V] The transport dropped the half that makes it persistent.**
|
||||
`src/main/ssh/relay-protocol.ts:34`:
|
||||
|
||||
```ts
|
||||
export const MessageType = { Regular: 1, KeepAlive: 9 } as const
|
||||
```
|
||||
|
||||
The 13-byte frame header was copied from a well-known persistent protocol, but
|
||||
only 2 of its 9 message types were kept. Dropped: `Control`, `Ack`, `Disconnect`,
|
||||
`ReplayRequest`, `Pause`, `Resume`, `None`. The surviving ACK field feeds only
|
||||
`unackedTimestamps` (`ssh-channel-multiplexer.ts:107,448,596`), a seq→timestamp map
|
||||
read solely by the death timer. **Nothing is retained at the transport layer;
|
||||
nothing is replayed.** Consequence: three layers above it each rebuilt
|
||||
retain-and-replay independently (SSH credit ledger, runtime-RPC source-range
|
||||
ledger, daemon batcher — the last deliberately lossy).
|
||||
|
||||
**[V] Method layer is transport-agnostic.** `runtime/rpc/methods/terminal.ts` and
|
||||
`git.ts` have 0 SSH references; `files.ts` has 27.
|
||||
|
||||
**[V] Three durable records exist, plus a fourth copy on the host.**
|
||||
|
||||
| | key | pane identity is |
|
||||
| ------------------------------------- | ------------------------- | ----------------------------- |
|
||||
| PTY binding (`WorkspaceSessionState`) | `(hostId, tabId, leafId)` | **the key** |
|
||||
| `SshRemotePtyLease` | `(targetId, relayPtyId)` | **optional payload** |
|
||||
| `SshPtyConsumerRecovery` | `targetId` | n/a (per-target resume token) |
|
||||
| relay `attachIdentity` | per PTY, in-memory | compare-only |
|
||||
|
||||
**[R] ~374 lines of pure lease↔binding reconciliation in `persistence.ts`**, and 9
|
||||
enumerated states where the records can disagree. Example of an independent latent
|
||||
bug: absence of a lease means "restorable" in `isRestorablePtyBinding` but "not
|
||||
restorable" in `hasRestorableSshRemotePtyLease` — so whether a binding survives
|
||||
depends on whether a _sibling leaf_ had a binding.
|
||||
|
||||
**[V] The relay holds nothing durably.** Its PTY table is
|
||||
`private ptys = new Map<string, ManagedPty>()` (`src/relay/pty-handler.ts:356`).
|
||||
No relay-side persistence of PTY state exists. `pty.serialize`/`revive` hand state
|
||||
to the _client_ and take it back; `revive` requires the original PIDs to still be
|
||||
alive. On PTY exit the entry is deleted with no tombstone.
|
||||
|
||||
**[V] Pane identity is write-only over the wire.** `listProcesses` and `attach`
|
||||
never return `paneKey`/`tabId`/`leafId`; `attach` only compares and throws
|
||||
`identity mismatch`. The one RPC that returns identity (`pty.serialize`) has
|
||||
**zero production callers** — test files only.
|
||||
|
||||
**[V] The client's lease is consulted from a synchronous, offline, disk-only path.**
|
||||
`hasRestorableSshRemotePtyLease` is called inside `setLocalWorkspaceSession`
|
||||
(`src/main/persistence.ts:6465`), a synchronous `void` method invoked on every
|
||||
renderer session publish including during quit (`:6287`, `:6351`).
|
||||
|
||||
## 3. Alternatives already refuted
|
||||
|
||||
**A. Merge the lease into the pane binding.** Blocked by two load-bearing
|
||||
properties: a lease must be able to name a live shell **no pane owns** (the actual
|
||||
STA-3077 field state; reattach enumerates these), and a lease **outlives** the
|
||||
binding — expiry deletes the binding and then authorizes recreating a shell for 30s
|
||||
(`SSH_PANE_RECOVERY_GRACE_MS`, `orca-runtime.ts:1771`).
|
||||
|
||||
**B. Delete the lease; make the remote host authoritative.** Refuted. The host has
|
||||
no durable state, cannot be queried for pane identity, has no notion of "recently
|
||||
died, replacement authorized", and is by construction unreachable exactly when the
|
||||
record is needed. Decisive: you cannot replace a synchronous local disk read during
|
||||
quit with an RPC to a host that is down.
|
||||
|
||||
**C. A second "authority architecture" beside the existing one** (+60,903 lines,
|
||||
394 new files). Rejected earlier: it fixed none of the three root causes.
|
||||
|
||||
## 4. The design under review
|
||||
|
||||
### Plane 1 — data plane (adopted from a separate proposal)
|
||||
|
||||
Restore a delivery guarantee at the transport, then collapse what was rebuilt above
|
||||
it. Roughly: capability handshake; one delivery-identity type; one credit-ledger
|
||||
implementation parameterised by unit and limit; one chunk-and-ack primitive; one
|
||||
lane vocabulary; binary PTY payloads reusing the binary terminal frame Orca already
|
||||
ships to mobile; prebuilt native binaries so first connect stops needing a compiler;
|
||||
then teach the relay to listen on a loopback WebSocket, forward that port, and let
|
||||
the _same_ client serve the SSH route — after which SSH is "launch + access" only.
|
||||
|
||||
### Plane 2 — control plane (proposed here; the reviewed contribution)
|
||||
|
||||
The lease answers two different questions with one record keyed for neither:
|
||||
|
||||
- "which shell does this pane own?" → wants a **pane** key
|
||||
- "which shells exist that nobody owns?" → wants a **pty** key
|
||||
|
||||
Proposal: split by question.
|
||||
|
||||
- **Ownership record**, keyed `(host, pane)` — the _same_ key local already uses.
|
||||
Transport-specific payload (lease state + timestamps for SSH; nothing extra for
|
||||
local). Makes "one pane, one live shell" structural rather than enforced.
|
||||
- **Orphan inventory**, keyed `(host, ptyId)` — live shells with no owner. Feeds the
|
||||
reattach work list and the disconnected-cleanup UI.
|
||||
- The 30s recovery grant becomes a terminal state on the pane-keyed record.
|
||||
- Offline consumers keep reading local disk synchronously, unchanged.
|
||||
|
||||
Claimed payoff: the ~209 lines of supersession/ranking/rollback/dual-matchers exist
|
||||
_only_ to bolt a pane-keyed constraint onto a pty-keyed record, and become
|
||||
unnecessary.
|
||||
|
||||
## 5. Historical defect, now resolved
|
||||
|
||||
This defect was resolved by the shared `bindPaneShell` producer used by relay reattach and both
|
||||
spawn handlers. Its mutation proof is recorded in
|
||||
[S5](./new-design-goalposts.md#s5--the-superseded-pane-fence-is-live-on-the-reattach-path--proven).
|
||||
|
||||
## 6. Hard constraints
|
||||
|
||||
- **Cross-platform**: macOS, Linux, Windows (+ WSL). No `echo $$`/`ps` assumptions.
|
||||
- **Folder workspaces**, not only git worktrees.
|
||||
- **Remote wire compatibility**: clients and remote hosts update independently;
|
||||
mixed versions are normal. New optional field = safe; new stream opcode must be
|
||||
capability-negotiated (decoders drop unknown opcodes silently); changing what a
|
||||
host publishes reaches old clients with no wire change.
|
||||
- **Git 2.25 baseline** for git-touching work.
|
||||
- Durable on-disk state is also read by pinned older builds in cross-version tests.
|
||||
- "Unknown is not dead": disconnect, timeout, absent inventory, and retry
|
||||
exhaustion must never authorize destroying a process.
|
||||
- No `max-lines` lint disables.
|
||||
|
||||
## 7. What the review must answer
|
||||
|
||||
1. **Correctness holes.** Which cases does the Plane 2 design get wrong or fail to
|
||||
cover? Specifically: multi-window / multi-device on one host; app restart while
|
||||
disconnected; relay restart; host reboot; PTY exit during disconnect; two panes
|
||||
racing for one shell; migration from existing on-disk state; a pane deleted while
|
||||
its shell lives; the same worktree open in two windows.
|
||||
2. **Simplicity.** Is there a _simpler_ design with the same correctness? Simpler
|
||||
means fewer code paths and fewer concepts, not merely fewer lines. Is the
|
||||
two-record split actually necessary, or is there a single-record formulation that
|
||||
survives the section-3 refutations?
|
||||
3. **Migration.** Re-keying a durable record needs a story for existing state,
|
||||
partial writes, and rollback.
|
||||
4. **Testability.** What oracle would fail if each guarantee were removed? A design
|
||||
whose guarantees cannot be falsified by mutation is not finished.
|
||||
5. **Is Plane 2 even the right cut?** Attack the premise that data plane and control
|
||||
plane are separable here.
|
||||
File diff suppressed because one or more lines are too long
-256
@@ -1,256 +0,0 @@
|
||||
FINAL DESIGN — one leaf-keyed ownership record; pane identity deleted from the attach path; death proved only by the relay process that spawned the shell
|
||||
|
||||
=====================================================================
|
||||
D0. SCOPE AND THE ONE RULE EVERYTHING FOLLOWS FROM
|
||||
=====================================================================
|
||||
Problem (from the brief): SSH reconnect grows pane cardinality (2 -> 19 -> 20, STA-3077); a coding agent gets resumed twice into one transcript (RC2); one PTY's failure can drop a whole host connection (RC3, out of scope — Plane 1).
|
||||
|
||||
One rule, applied without exception:
|
||||
|
||||
Ownership ("which shell does this pane own?") is DURABLE, CLIENT-LOCAL, and read
|
||||
synchronously from disk while offline. Evidence about a shell's FATE is
|
||||
NON-DURABLE, HOST-LOCAL, and must be REPORTED by the host, never INFERRED by the
|
||||
client from timing, from the absence of an entry, or from the shape of an error
|
||||
string.
|
||||
|
||||
Every defect found across three review rounds is one violation of the second clause. The design's job is to remove all of them, not to add an authority layer.
|
||||
|
||||
SCOPE DECLARATION (grok, low): the record is host-agnostic (`hostId` is a field). The _death rule_'s row 2 is relay-specific. Local and daemon panes reach `terminated` only through row 1 (an exit observed in-process, which is how they already work) and never through row 2. That is stated rather than left implicit.
|
||||
|
||||
=====================================================================
|
||||
D1. THE DURABLE RECORD
|
||||
=====================================================================
|
||||
|
||||
```ts
|
||||
/** Main-owned. TOP-LEVEL in PersistedState — never inside WorkspaceSessionState. */
|
||||
type PaneShellOwnership = {
|
||||
leafId: string // PRIMARY KEY. Terminal-layout leaf UUID.
|
||||
hostId: ExecutionHostId // FIELD: 'local' | 'ssh:<target>' | 'runtime:<id>'
|
||||
worktreeId: string // FIELD. Opaque FULL string incl. any `::workspace:<uuid>` suffix.
|
||||
ptyId: string // RELAY-NATIVE form for SSH (toRelaySshPtyId). One namespace, always.
|
||||
incarnationId: string // REQUIRED. Relay mints one per pty (relay/pty-handler.ts:1525).
|
||||
relayInstanceId?: string // The relay PROCESS that owned it, as reported at bind time.
|
||||
state: 'attached' | 'detached' | 'terminated'
|
||||
createdAt: number
|
||||
updatedAt: number
|
||||
lastAttachedAt?: number
|
||||
lastDetachedAt?: number
|
||||
}
|
||||
// Storage: PersistedState.paneShellOwnershipByLeafId: Record<string, PaneShellOwnership>
|
||||
```
|
||||
|
||||
Note what is NOT here: no `tabId`, no `paneKey`, no `hostEpoch`, no `resumed`, no second (pty-keyed) record.
|
||||
|
||||
WHY `leafId` AND NOT `paneKey`. `makePaneKey(tabId, leafId)` returns `${tabId}:${leafId}` and the repo states at src/shared/stable-pane-id.ts:47-49 that "only the leaf UUID is remint-stable pane identity (the tab half changes on pane break-out)" — which is why `isEquivalentPaneKey` exists at :50-57. The mutation is a shipping gesture: `detachTerminalPaneToTab` (src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.ts:245-310) moves a LIVE pane and its ptyId into a new tab via `createTab(..., initialPtyId)` while explicitly not killing the PTY (:275-276). `leafId` is a UUID and is never remapped once it is one (src/main/persistence.ts:1858-1872 skips ids already passing `isTerminalLeafId`). Keying by the leaf deletes the transfer problem instead of managing it.
|
||||
|
||||
WHY NO `tabId` AT ALL (this is the round-3 change). A stored tabId is either frozen (and then it names a tab the leaf no longer lives in) or updated (and then it desynchronizes from the relay's frozen copy). Both are live production failures — see D2. The tab is LOCATION, derived from the live layout at the moment it is needed; it is never stored and never trusted.
|
||||
|
||||
WHY TOP-LEVEL. `session:set` / `session:set-sync` (src/main/ipc/session.ts:13, :31) accept a whole `WorkspaceSessionState` from a renderer with no revision or fencing token; the clobber guards at src/main/persistence.ts:6379, :6384-6476 exist only because of that. `sshRemotePtyLeases` is already top-level. The KEY is shared with the layout; the STORAGE LOCATION is not.
|
||||
|
||||
UNIQUENESS. At most one non-terminal record may name a given `(hostId, ptyId, incarnationId)`, enforced on write through a reverse index. Not ptyId alone: leases store the relay-native id (`getRelayPtyIdForSshLeaseStorage` -> `toRelaySshPtyId` strips the connection scope, src/main/persistence.ts:6641-6643, src/shared/ssh-pty-id.ts:58-67) and relay ids are `pty-${this.nextId++}` with `nextId = 1` per relay PROCESS (src/relay/pty-handler.ts:356-357, :1440-1443), so `pty-1` recurs on every host after every relay restart. Round 1's rejection ("a cross-target collision on ptyId is impossible") is withdrawn; it is guaranteed, not impossible.
|
||||
|
||||
=====================================================================
|
||||
D2. DELETE PANE IDENTITY FROM THE ATTACH PATH (the largest round-3 change)
|
||||
=====================================================================
|
||||
Both reviewers independently attacked the attach-time identity comparison. Verified, and it is worse than either stated: the apparatus is simultaneously over-strict and under-strict, and it is a live duplicate-agent bug today.
|
||||
|
||||
VERIFIED FACTS
|
||||
|
||||
- The relay compares frozen strings: `attachIdentityMismatches` (src/relay/pty-handler.ts:334-339) does full-string `expected.paneKey !== managed.paneKey || expected.tabId !== managed.tabId`; `attachIdentity` is captured once at spawn from `ORCA_PANE_KEY`/`ORCA_TAB_ID` (:1512-1517, :1446). On mismatch it throws `PTY "<id>" not found (identity mismatch)` (:1616-1626).
|
||||
- `isSshPtyNotFoundError` is a message regex `/PTY ".+" not found/i` (src/main/providers/ssh-pty-errors.ts:12-15), so it is TRUE for the mismatch string. `reattachSshPtySession` re-emits it as `SSH_SESSION_EXPIRED: <id> SSH_PTY_IDENTITY_MISMATCH` (src/main/providers/ssh-pty-session-reattach.ts:218-226).
|
||||
- The renderer's `isProvenSshSessionGoneError` returns TRUE for anything containing `SSH_SESSION_EXPIRED` and for the raw not-found regex, and IGNORES the mismatch marker (src/renderer/src/components/terminal-pane/reattach-failure-classification.ts:24-30). Its two call sites then clear the binding and call `startFreshColdRestoreAgentResume` (pty-connection.ts:8839-8847, :9086-9097) — a second shell and a SECOND AGENT RESUME while the original shell keeps running. That is RC2, on a live process, reachable from a shipped gesture.
|
||||
- The spawn-path reattach builds the expected identity from the CURRENT options (`args.options.paneKey ?? env.ORCA_PANE_KEY`, ssh-pty-session-reattach.ts:66-68), i.e. the current tab — so after a detach it mismatches. The connect-path reattach builds it from the lease's stored tabId (`expectedIdentityForLease`, src/main/ssh/ssh-relay-session.ts:190-210, consumed at :2201-2210), which is never re-keyed on detach — so it matches, attach succeeds, and then `persistPtyBinding(tabId = old tab, mayCreate:false)` refuses because the leaf is no longer in that tab's layout (src/main/persistence.ts:6821-6877 sets `terminalMembershipChanged` and returns false), leaving the shell running unbound (ssh-relay-session.ts:2513-2521). Grok's "scissors" is confirmed on both blades.
|
||||
- The apparatus does not even do its own stated job ("generation resets can reuse PTY IDs; reject conflicting identities", pty-handler.ts:1615): after a relay restart that recycles `pty-3` for the SAME pane, paneKey and tabId both match and a different shell is accepted.
|
||||
|
||||
RESOLUTION — delete the comparison; do not replace it with another comparison.
|
||||
|
||||
1. Relay: delete `attachIdentity`, `attachIdentityMismatches`, the `params.expectedPaneKey`/`expectedTabId` handling and the `(identity mismatch)` throw. `ManagedPty.paneKey`/`tabId` STAY — they are documented as separate from attach identity (pty-handler.ts:139-143) and are used by the exit listener (:765) and, newly, by listProcesses reporting (D4). paneKey stops being an authorization INPUT and becomes a reported HINT.
|
||||
2. Main: delete `ExpectedPtyIdentity`, `expectedIdentityForLease`, `expectedIdentityByPtyId`, the `expectedIdentity` parameter threaded through `attachPtyWithRetry`, `isSshPtyIdentityMismatchError`, `SSH_PTY_IDENTITY_MISMATCH_ERROR`, the marker append, and the mismatch branch at ssh-relay-session.ts:2628-2634.
|
||||
3. Renderer: `isProvenSshSessionGoneError` keeps only proofs. (After step 1/2 the mismatch string ceases to exist; the renderer fix ships FIRST and independently, because it must protect users running against relays that have not been redeployed.)
|
||||
4. Recycled-id rejection moves to two places that already hold exact data:
|
||||
(a) PROCESS GATE, once per connection: if the owner grant reports a `relayInstanceId` different from the one recorded on a record, that record's ptyId is not attachable at all. One comparison covers every pty on the host. Within a single relay process ids never recycle (`nextId` is monotonic), so this is complete against new relays.
|
||||
(b) PER-ATTACH GUARD, zero wire change: EVERY `pty.attach` success path already returns `incarnationId` (src/relay/pty-handler.ts:1646, :1653, :1668, :1676), and the client already parses and remembers it (`requestSshPtyAttach` -> `rememberPtyIncarnation`, ssh-pty-session-reattach.ts:74-100). Compare it to `record.incarnationId`; on mismatch, throw — the existing `activationLease.rollback()` already runs on a thrown error — and classify `detached`, never death. `RecentPtyOutputBuffer.read()` is explicitly non-mutating (src/main/runtime/recent-pty-output-buffer.ts:75-85), so abandoning a wrongly-attached shell does not consume its replay.
|
||||
This is strictly stronger than the frozen-string check (a per-pty UUID vs a pane label), needs NO new request field, and works against OLD relays — which a new `expectedIncarnationId` request parameter would not.
|
||||
5. The reattach BIND resolves the tab from the live layout: find the tabId whose `terminalLayoutsByTabId[*].root` contains this `leafId`. If no layout contains it, refuse exactly as today (leave the shell running unbound, "no durable pane owns this"). `mayCreate:false`'s guarantee is preserved; its dependence on a stored tabId is deleted.
|
||||
|
||||
ONE IDENTITY RULE, EVERYWHERE: the leaf is identity; the tab is derived location; the incarnation is shell identity; the relay instance is process identity. There is no fourth.
|
||||
|
||||
=====================================================================
|
||||
D3. THE DEATH RULE — two rows, one new field
|
||||
=====================================================================
|
||||
`resumed` and `hostEpoch` are deleted and must not be built.
|
||||
|
||||
- `resumed: replaces !== null` (src/shared/pty-consumer-session.ts:275); `expireOwner()` nulls the incumbent after `PTY_CONSUMER_OWNER_GRACE_MS = 30_000` (:281-287, contract :2), after which admission takes `if (!current) return this.newOwner(...)` (:215-216) and `resumed` is false regardless of proof. Unreachable for quit, sleep, overnight, reboot — the normal case. Its own contract comment says it means "the client's checkpoints for the previous claim no longer apply" (pty-consumer-session-contract.ts:49-52): a delivery-checkpoint flag, not liveness.
|
||||
- `hostEpoch = Date.now() - os.uptime()*1000` under exact equality is a reconstructed estimate; second-granularity uptime, NTP steps and suspend/resume make it differ on an unrebooted host, and "differs" was the grant-YES row — it fails OPEN toward respawn. Repo precedent settles it: `readCurrentDaemonReadyIdentity` reads `/proc/sys/kernel/random/boot_id` on Linux only and returns bare `{ startedAtMs }` elsewhere (src/main/daemon/daemon-ready-identity.ts:11-13).
|
||||
|
||||
THE ONE NEW GRANT FIELD: `relayInstanceId?: string` on `PtyConsumerSessionGrant` (src/shared/pty-consumer-session-contract.ts:43-59) — a `randomUUID()` minted once at relay process start (`randomUUID` is already imported at relay/pty-handler.ts:6). It is genuinely new information: the grant's existing `serverBuildId` is `launchVersion` (src/relay/relay.ts:694-696), a build string that is identical across restarts of the same build.
|
||||
|
||||
| Condition | Transition | Respawn grant |
|
||||
| ---------------------------------------------------------------------------------- | ------------ | -------------------- |
|
||||
| PTY exit observed on a live attached stream | `terminated` | yes |
|
||||
| attach returns not-found AND the grant's `relayInstanceId` EQUALS the recorded one | `terminated` | yes (subject to E-1) |
|
||||
| attach returns an incarnation different from the recorded one | `detached` | no |
|
||||
| `relayInstanceId` absent on either side, or differing | `detached` | no |
|
||||
| any other error (today's `restoreRequired` path, ssh-relay-session.ts:2612-2620) | `detached` | no |
|
||||
|
||||
Row 2 is sound and UNBOUNDED IN TIME: `this.ptys` is the relay process's only store (pty-handler.ts:356) and entries are deleted on exit, so "the same process that minted this pty no longer has it" means it exited. It is sound on Windows too, because the relay OBSERVED the exit — unlike any rule that reasons from a relay restart. A restarted relay reports a different id and we correctly claim no knowledge.
|
||||
|
||||
SHIP GATE (grok H4): row 2 grants a respawn on a not-found detected by a message regex. One shared normalization helper (`toRelaySshPtyId`) must be applied on every attach / mark / record / comparison path, and the record stores exactly one namespace (D1), so a namespace mismatch cannot manufacture a false not-found. Row-2 grant-yes does not ship until E-1 is green.
|
||||
|
||||
STOP FABRICATING THE EXIT. `handlePtyReattachFailure` currently sends `pty:exit { code: -1 }` (ssh-relay-session.ts:2643-2647) plus `clearProviderPtyState` + `deletePtyOwnership` + `markSshRemotePtyLease('expired')` on a plain not-found. On every `detached` row this is a lie about a process the client knows nothing about. Remove the whole block and route the unknown case into the branch that ALREADY EXISTS three lines above it: `pending.restoreRequired = 'reattachAttemptsExhausted'` + `wakeRecovery` (:2612-2620), which is non-destructive by construction. This is a branch COLLAPSE, not a new signal — which is also the answer to the objection that removing only the send turns a wrong signal into no signal.
|
||||
|
||||
=====================================================================
|
||||
D4. ORPHANS — a connect-time projection, never an authority
|
||||
=====================================================================
|
||||
|
||||
```
|
||||
orphans(host) = successful listProcesses(host)
|
||||
MINUS { (ptyId, incarnationId) named by a non-terminal record on this host }
|
||||
```
|
||||
|
||||
Computed at connect/reattach, held in memory for the connection's life, discarded on disconnect. No durability, no migration, no rollback.
|
||||
|
||||
WHAT `listProcesses` IS: it iterates only `this.ptys` (pty-handler.ts:1894-1911) — the current relay process's Map. No durability, no OS scan. It is therefore (i) the cleanup-UI source and (ii) a SUPPLEMENT to the reattach work list, never the sole discovery authority. The work list stays `getPtyIdsForConnection(target) UNION` the non-terminal records (today's `leasedPtyIds` shape, ssh-relay-session.ts:2211-2219).
|
||||
|
||||
ABSENCE IS NEVER AUTHORITATIVE. The repo already adjudicated this with an issue number: `pty:listSessions` documents "Absence is authoritative only from a provider that serializes claims — otherwise it is 'unknown', never 'absent' (#8459)" (src/main/ipc/pty.ts:7343-7347), and `SshPtyProvider.hasPty` returns `null` before a completed listing "a miss there is ignorance about the host, not a dead PTY" (src/main/providers/ssh-pty-provider.ts:308-312). The only aggregating caller launders every remote failure with `provider.listProcesses().catch(() => [])` (ipc/pty.ts:7333), and `PtyProcessListAdmission.admit` throws on capacity and on `agent_session_ownership_unknown` (src/main/providers/pty-process-list-admission.ts:60-99). So: a rejected or truncated listing yields orphans = UNKNOWN (empty UI, no claims, no retirement), never orphans = everything.
|
||||
|
||||
THE ONE NEW RESPONSE FIELD: `PtyProcessSummary` gains optional `paneKey`. The relay already holds it (`ManagedPty.paneKey`, pty-handler.ts:1446, :139-140) and omits it from the summary (:1899-1908). Additive optional response field — safe under docs/reference/remote-wire-compatibility.md, no opcode, no negotiation. It does NOT reach the client for free: `PtyProcessInfo` has no `paneKey` (src/main/providers/pty-process-info.ts:4-16) and `admit` REBUILDS an allowlist (pty-process-list-admission.ts:106-115). Three edits — relay summary, `PtyProcessInfo`, allowlist — and the oracle asserts survival THROUGH `admit`. `incarnationId` and `worktreeId` already traverse that whole chain, which is what makes the incarnation-scoped subtraction free.
|
||||
|
||||
ADOPTION matches `parsePaneKey(reported.paneKey)?.leafId` — never the full string, because the relay's copy is frozen at spawn and names a tab the leaf may have left. Because the record is leaf-keyed this is a direct lookup, not fuzzy equivalence; `isEquivalentPaneKey` is not imported here. Adopt is a CAS: write `leafId -> (ptyId, incarnationId)` only if no non-terminal record exists for that leaf AND none on this host names that `(ptyId, incarnationId)`. Exactly one of two concurrent adopts wins; the loser gets a typed refusal.
|
||||
|
||||
KILL POLICY (replaces the "owned elsewhere" classification, which is DELETED). Both reviewers attacked `worktreeId`-based classification from opposite directions and both are right, because `worktreeId = <repoId>::<path>` and `repoId` is a client-local `randomUUID()` minted at repo add (src/shared/worktree-id.ts:15-28; src/main/runtime/orca-runtime.ts:18729). So "unknown worktreeId" means "not in this install's current state file": it wrongly EXEMPTS the user's own shells after a repo re-add, profile transfer or state reset, and it wrongly INCLUDES another device's shells whenever the two installs share state. One rule replaces it: the orphan list never auto-kills, never bulk-kills, and never kills as a side effect of any reattach or cleanup pass. Kill is an explicit per-shell user action, with cwd, title, worktreeId and reported paneKey shown. No classifier, no new field, and it is safe in both directions.
|
||||
|
||||
OLD-RELAY DEGRADATION: a relay that omits `paneKey` yields unclassified orphans that are listed and never killed — exactly today's behavior.
|
||||
|
||||
=====================================================================
|
||||
D5. RECORD LIFETIME — three exits, none of them "absent from a list"
|
||||
=====================================================================
|
||||
|
||||
1. RETIRE (delete). The record reached `terminated` (row 1 or row 2 of D3, or explicit user close via `retireExitedPty`, ssh-relay-session.ts:2153-2172). Terminal records are garbage-collected after a bounded retention (long enough to serve any surviving grant window) so they cannot mask a recycled `pty-1`. There is NO retire-on-absence clause: the case it was written for (shell exited during a disconnect, relay still up) is already covered by row 2 — the attach itself proves the exit — so deleting the clause costs nothing and removes a direct contradiction with #8459.
|
||||
2. RELEASE (keep the shell, drop the claim). Fired ONLY from the explicit pane-close and tab-delete code paths — never from "this publish did not contain the leaf". Detach is not atomic across layouts: `createTab` installs an empty layout, then `setTabLayout(sourceTabId, ...)` removes the leaf, then `setTabLayout(tab.id, ...)` adds it (terminal-pane-tab-detach.ts:298-300), so between the last two no layout contains the leafId, and any `session:set` flush in that window would look like a durable removal. Released records are cleared to a non-claiming state: the shell becomes orphan-eligible (visible, adoptable) and is deliberately NOT `terminated`, so it can never authorize a respawn.
|
||||
3. EXPLICIT USER CLOSE -> `terminated`, and the record dies with the shell.
|
||||
|
||||
RESIDUAL, STATED HONESTLY: a shell that died while the relay ALSO restarted leaves a `detached` record with no proof available. It stays restorable until the user acts. That is the conservative direction and it is bounded by the affordance below, not by a timer.
|
||||
|
||||
PRODUCT AFFORDANCE, REQUIRED IN THE SAME PR. A `detached` pane renders as disconnected with two explicit actions — "reattach (retry)" and "start a new shell" (the latter retires the record). Nothing infers death, nothing auto-spawns. Per AGENTS.md this must follow docs/STYLEGUIDE.md. COPY CONSTRAINT: the UI must never assert that a shell is dead. The claim "after a relay crash the shells are almost certainly dead" is REMOVED from this design: it rests on POSIX master-fd close SIGHUPing the foreground group, and on Windows the relay is deliberately launched through WMI because "Windows sshd kills the exec channel's process tree on close" (src/main/ssh/ssh-relay-deploy.ts:1805), with ConPTY and no SIGHUP. No rule in this design concludes anything from a relay restart, so the design is correct either way; only the copy had to change.
|
||||
|
||||
=====================================================================
|
||||
D6. HISTORICAL — one partition per (target, pane). SUPERSEDED.
|
||||
=====================================================================
|
||||
This migration proposal was rejected. The shipped containment keeps the existing persistence
|
||||
planes and gives the pane binding one local home; see [S4](./new-design-goalposts.md#s4--one-partition-per-target-pane--proven).
|
||||
|
||||
SSH pane bindings live in two partitions today. Readers and writers that consult ONLY `ssh:<target>`: `resolvePersistedStablePaneOwner` (src/main/ipc/pty.ts:676-677), `retirePersistedStablePaneOwner` (:760-761, :783), the CAS write `persistPtyBinding(..., expectedBinding)` (:833), and the two spawn upserts (:5108, :6493). Against that, the relay reattach write has no hostId (ssh-relay-session.ts:2500-2511 -> `resolveHostId(undefined)` -> `LOCAL_EXECUTION_HOST_ID`, persistence.ts:6243-6246) and the renderer keeps SSH worktrees in `local` deliberately (workspace-session-host-persistence.ts:167-173). `durablyBoundPtyIdForPane` hedges ssh-first-then-local (persistence.ts:7269-7277), so `supersedeSiblingLeasesForPane` early-returns when the bound pty differs (:7229-7232) — supersession silently no-ops and both leases stay live. That is the STA-3077 mechanism itself.
|
||||
|
||||
FIX: one accessor `sessionForSshPaneBindings(connectionId): WorkspaceSessionState` returning the `local` partition, routed through by EVERY reader and EVERY writer above, flipped in a single atomic PR — not five write-site edits. `local` wins because the only publisher of pane membership writes there and `mayCreate:false` is evaluated there. A one-time fold moves `ssh:<target>.terminalLayoutsByTabId[*].ptyIdsByLeafId` and `tabsByWorktree[*].ptyId` into `local`, preferring `local` on conflict; the loser is NOT demoted to orphan-candidate by ptyId alone (recycled ids collide) — it carries `incarnationId` or is dropped.
|
||||
|
||||
=====================================================================
|
||||
D7. HISTORICAL — one bind producer. RESOLVED.
|
||||
=====================================================================
|
||||
The shared `bindPaneShell` producer now serves relay reattach and both spawn handlers; see
|
||||
[S5](./new-design-goalposts.md#s5--the-superseded-pane-fence-is-live-on-the-reattach-path--proven).
|
||||
|
||||
`rememberPaneKeyForPty` (ipc/pty.ts:517-525) has two callers, both spawn (:5222, :6668). `restoreReattachedPtyRuntime` calls `runtime.registerPty` instead (ssh-relay-session.ts:2523), and `isSupersededPtyId` returns `false` for an unrecorded id by design (:284-296) — so the shipped superseded-PTY fence is INERT on the path it was built for. Collapse to one `bindPaneShell({ hostId, worktreeId, leafId, ptyId, incarnationId, relayInstanceId })` that, in one call, (a) resolves the current tab containing the leaf (refusing if none), (b) writes the durable record, (c) writes the in-memory `ptyPaneKey`/`paneKeyPtyId` fence maps using the paneKey composed from the CURRENT tab, (d) calls `registerPty`. All three paths call it and nothing else.
|
||||
|
||||
=====================================================================
|
||||
D8. COLLAPSE THE RESTORABLE FORK — and the offline/synchronous path
|
||||
=====================================================================
|
||||
`setLocalWorkspaceSession` forks on whether the INCOMING layout map is empty (persistence.ts:6450-6473): empty -> `isRestorablePtyBinding` (absence of a lease implies restorable, :6620-6631); partial -> `hasRestorableSshRemotePtyLease` (requires a live lease, :6670-6685). They disagree on the absent-record case, so whether a binding survives depends on whether a SIBLING LEAF had one.
|
||||
|
||||
One predicate replaces both: RESTORABLE IFF `paneShellOwnershipByLeafId[leafId]` exists with state `attached` or `detached`.
|
||||
|
||||
OFFLINE / SYNCHRONOUS ACCESS PATH (refutation B, honored exactly). `setLocalWorkspaceSession` is a synchronous `void` method invoked on every renderer publish including during quit (persistence.ts:6465, :6287, :6351). The loop at :6448-6462 already iterates `priorLayout.ptyIdsByLeafId` and therefore already holds a `leafId`, so the new predicate is a DIRECT map lookup on a top-level, main-owned, in-memory-mirrored disk record: no key composition, no lease-array scan, no `incomingHasAnyBinding` branch, no mux, no RPC, no host. It works with the host down and during quit — which is precisely what refutation B says cannot be replaced by asking the host.
|
||||
|
||||
=====================================================================
|
||||
D9. THE REATTACH ALGORITHM (SSH, on connect)
|
||||
=====================================================================
|
||||
|
||||
1. Obtain the consumer owner grant. Read `grant.relayInstanceId` (may be absent — old relay).
|
||||
2. Work list = `getPtyIdsForConnection(target)` UNION `{ r.ptyId | r.hostId === 'ssh:'+target and r.state !== 'terminated' }`, every id normalized through `toRelaySshPtyId`.
|
||||
3. PROCESS GATE. For each record: if both `grant.relayInstanceId` and `r.relayInstanceId` are present and differ, do not attach by that id. Set `detached`. Its shell, if any, is rediscoverable only through step 5.
|
||||
4. For each remaining id, call `pty.attach { id, cols, rows }` — with NO expectedPaneKey and NO expectedTabId.
|
||||
4a. SUCCESS -> if a record exists and `response.incarnationId !== r.incarnationId`, throw (activation lease rolls back), set `detached`, no grant, no exit. Otherwise call `bindPaneShell(...)`: resolve the current tab containing `leafId`; if none, log and leave the shell running unbound (today's refusal, preserved); else write record (`attached`, `relayInstanceId = grant.relayInstanceId`), fence maps, `registerPty`.
|
||||
4b. NOT-FOUND -> if `grant.relayInstanceId` is present and equals `r.relayInstanceId`: `terminated`, retire, grant authorized (gated on E-1). Otherwise: `detached`, no grant, and route into the existing `restoreRequired = 'reattachAttemptsExhausted'` branch. No `pty:exit -1`, no `clearProviderPtyState`, no `deletePtyOwnership`.
|
||||
4c. ANY OTHER ERROR -> `detached` via the same existing branch.
|
||||
5. Call `listProcesses` best-effort. If it REJECTS or admission truncates it: orphans = unknown; do nothing. If it RESOLVES: orphans = returned MINUS `(ptyId, incarnationId)` named by non-terminal records on this host. For each orphan with a reported `paneKey`, look up `parsePaneKey(paneKey)?.leafId`; if that leaf exists in the live layout and has no non-terminal record, CAS-adopt it via `bindPaneShell`. The rest are listed in the cleanup UI, never auto-killed.
|
||||
|
||||
=====================================================================
|
||||
D10. MIGRATION, SEQUENCING, ROLLBACK
|
||||
=====================================================================
|
||||
ORDER — each its own PR, each independently revertible:
|
||||
|
||||
| | Step | Contents |
|
||||
| --- | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| A | Identity-mismatch is not death | Renderer: `isProvenSshSessionGoneError` stops treating the mismatch marker as proof. Relay+main: delete the whole attach-identity apparatus (D2). Pure bug fix, no design dependency, ships FIRST; the renderer half must ship even before relays are redeployed. |
|
||||
| E-0 | Stop lying about exits | Collapse the plain-not-found branch of `handlePtyReattachFailure` into the existing non-destructive `restoreRequired` branch (D3). Ships with, or immediately followed by, the D5 disconnected-pane affordance. |
|
||||
| P | Single partition | D6: one accessor for every reader and writer, plus the fold migration, atomically. |
|
||||
| F | Single bind producer | D7: `bindPaneShell`; makes the shipped fence live on the reattach path; introduces tab-from-layout resolution. |
|
||||
| E-1 | Prove the grant executes | Production-id-shape oracle for `recoverTerminalPane`; record the actual behavior. |
|
||||
| E-2 | Death rule | `relayInstanceId` on the grant + the D3 table. Conditional on E-1. |
|
||||
| W | Projection | `paneKey` through relay -> `PtyProcessInfo` -> admission allowlist; incarnation-scoped subtraction; explicit-only kill (D4). |
|
||||
| K | Re-key | The leaf-keyed record + D8 + D5 lifetime + `reassignSshTargetId` field rewrite + porting all lease readers. |
|
||||
|
||||
E-1 CONTEXT (why E-2 is conditional). The chain `not-found -> expired -> 30s grant -> createTerminal` has one spawn-authorizing caller, `recoverTerminalPane` (src/main/runtime/orca-runtime.ts:16449-16490), gated on `getRecentExpiredSshLease(...)`, whose comparison is raw `lease.ptyId === ptyId` with no normalization (:6310). `lease.ptyId` is relay-native (persistence.ts:7184) while the graph's `pty.ptyId` is app-form `ssh:<conn>@@pty-N` (ssh-relay-session.ts:2523), so for a real SSH pane they can never be equal, and the branch also cannot be reached by a local pane (it requires an SSH lease). The covering test seeds both sides as `'pty-expired'` and registers with a NULL connectionId (orca-runtime.test.ts:1350-1370, :2932-2940) — a shape production cannot produce. Do not build grant-arbitration machinery before E-1 shows the branch executing.
|
||||
|
||||
FORWARD MIGRATION AT K, per `SshRemotePtyLease`:
|
||||
|
||||
- has `leafId` -> one record. `state`, `createdAt`, `updatedAt` copied VERBATIM (any surviving grace window reads `updatedAt`). `worktreeId` copied as an OPAQUE FULL STRING — the `::workspace:<uuid>` folder-workspace suffix must survive, since matching is full-string equality (persistence.ts:7239, orca-runtime.ts:6308) and issue #12474 was a comparator that stripped exactly that. `ptyId` stays relay-native. `tabId` is DROPPED (D1). `relayInstanceId` absent -> the record falls to `detached` on first reconnect with no ambiguity to resolve. `incarnationId` absent on a legacy row -> treat as adoption-eligible rather than as a subtracting claim, so it cannot mask a recycled id.
|
||||
- no `leafId` -> no record; recovered at first reconnect by the leaf-half `paneKey` match, or listed as an orphan.
|
||||
- ALSO synthesize a record from any live layout binding (`ptyIdsByLeafId`) whose ptyId has no lease, so a bound-but-unleased pane is not silently demoted.
|
||||
- Sequence AFTER the legacy numeric-leaf remap (persistence.ts:1858-1872) so every record keys on a UUID.
|
||||
|
||||
PORT EVERY LEASE READER — the "write-only legacy projection" claim was FALSE and is corrected. Verified production readers of `sshRemotePtyLeases` outside persistence.ts:
|
||||
|
||||
- `getRecentExpiredSshLease` (orca-runtime.ts:6295-6316) with `ptyId` UNDEFINED, feeding `hasRecentExpiredSshLeasePane` (:6318-6323) -> the headless-mobile terminal-tab filter (:5290-5299 region, the `onlyRuntimeOwnedTerminals` filter) and `hasLiveOrPersistedServeOrSshOwnedPtyBinding` (:6355-6360). These are a live VISIBILITY signal for paired mobile/HUB clients and are NOT killed by the namespace bug, because they pass `ptyId` undefined. Port: "a non-terminal record names this leaf" replaces "an `expired` lease within 30s". Visibility becomes bounded by the D5 lifetime rule instead of a timer; that behavior change is deliberate and must be covered by an oracle.
|
||||
- `ssh:terminateSessions` (src/main/ipc/ssh.ts:1282-1300) enumerates leases and distinguishes owned from expired (#2626). This is the EXISTING kill affordance; the D4 orphan UI FEEDS it rather than duplicating it. It is re-expressed over records: non-terminal records plus projection entries, with kill always explicit (D4).
|
||||
- `ssh:resetRelay` (ipc/ssh.ts:1370-1385) marks every live lease `expired` after force-killing the relay. Ported to: mark records `detached` (never `terminated` — the client killed the relay, which proves nothing about the shells). BEHAVIOR CHANGE, flagged for the owner: a user-initiated relay reset stops silently re-showing panes for 30s and instead shows the disconnected affordance.
|
||||
- `ssh-relay-session.ts:1914, :2197` internal lease reads -> record reads.
|
||||
|
||||
CROSS-VERSION READS. Pinned older builds read `sshRemotePtyLeases`. For ONE release the new writer emits a derived legacy projection of the array. It must emit RELAY-NATIVE ptyIds via `toRelaySshPtyId` — leaking app-form ids gives pinned builds leases matching nothing, a silent no-op that looks like a passing test. Because the record already stores the relay-native form, this is a copy, not a conversion. It carries no `tabId`-dependent semantics, so old builds that call `expectedIdentityForLease` simply get `null` and attach without an identity — which, after step A, is the correct behavior anyway.
|
||||
|
||||
PARTIAL WRITE / ROLLBACK. Reuse the existing discipline (persistence.ts:7346-7360): snapshot, mutate in memory, `flushOrThrow`, undo on failure. One record, so this is a single-record transaction. Rollback of K: the legacy projection lets an older build boot on new state; the one-way loss is leafId-less rows, which the old build also could not act on.
|
||||
|
||||
`reassignSshTargetId` (persistence.ts:7055-7086) today re-keys partitions, rewrites lease `targetId`s and drops recoveries. Because `hostId` is a FIELD and not part of the key, it only rewrites a field in place plus the reverse index; no re-keying.
|
||||
|
||||
=====================================================================
|
||||
D11. ORACLES — each must redden under a stated mutation
|
||||
=====================================================================
|
||||
|
||||
1. DETACH-TO-TAB OVER SSH, THROUGH THE REAL RELAY COMPARISON. Bind an SSH pane in tab A leaf L, run `detachTerminalPaneToTab` to tab B, disconnect, reconnect. Assert: exactly one shell on the host, no second agent resume, the pane reattaches and the ptyId lands in tab B's layout. THIS MUST BE RED BEFORE STEP A. If it is green today, the harness stubbed the relay comparison or omitted `ORCA_PANE_KEY`/`ORCA_TAB_ID` from the spawn env, and the whole detach oracle family is measuring nothing. Gate the program on it going red first.
|
||||
2. Identity mismatch is not proof of death. Assert `isProvenSshSessionGoneError` is false for the mismatch string, no record reaches `terminated`, no grant. Mutation: restore `.includes(SSH_SESSION_EXPIRED)` as the sole predicate (reattach-failure-classification.ts:29).
|
||||
3. Wrong-incarnation attach is abandoned, not adopted and not fatal. Relay returns a different `incarnationId` than recorded: assert no bind, no stream, `detached`, no grant, activation lease rolled back. Mutation: skip the response check.
|
||||
4. Same-instance not-found is proof; different or absent is not. Two runs, recorded `relayInstanceId` equal -> `terminated` + grant; differing/absent -> `detached` + no grant. Mutation: flip the comparison.
|
||||
5. Failed listing is not an empty listing. Make `listProcesses` reject, and separately make `admit` throw `agent_session_ownership_unknown`: assert zero records retired and zero orphans reported. Mutation: replace the failure with a resolved `[]` — must redden, proving the implementation does not reuse the `.catch(() => [])` shape at ipc/pty.ts:7333.
|
||||
6. No fabricated exit on unknown. Drive `handlePtyReattachFailure` with a not-found and a differing `relayInstanceId`: assert `detached`, no grant, no `pty:exit -1`, and that provider state and ownership are NOT cleared. Mutation: restore the block at ssh-relay-session.ts:2641-2647.
|
||||
7. Detach mid-publish does not release. Force a `session:set` flush between `setTabLayout(source)` and `setTabLayout(target)`: assert ownership is NOT released. Mutation: release on any publish missing the leaf.
|
||||
8. Pane/tab delete releases. Delete a tab while its shell is live: assert the record is released, the shell IS an orphan, and no respawn is authorized. Mutation: remove the release rule — the shell must become invisible to both adopt and the cleanup UI.
|
||||
9. Production id shapes in the recovery grant (E-1). Seed relay-native `pty-7`; register the runtime pty as `ssh:<conn>@@pty-7` with a NON-NULL connectionId; call `recoverTerminalPane`; assert the actual behavior. Mutation: swap either namespace.
|
||||
10. E-2 false not-found. Same `relayInstanceId`, not-found caused by the wrong id form while the shell is in the Map under the correct id: assert NO `terminated` and NO grant. Mutation: skip `toRelaySshPtyId` on the attach path.
|
||||
11. Projection is incarnation-scoped. Record `pty-1@incX`; restart the relay; it mints `pty-1@incY` for a different pane: assert the fresh shell IS an orphan and the stale record does NOT mask it. Mutation: subtract on ptyId alone.
|
||||
12. `paneKey` survives `admit`. Mutation: remove it from the allowlist rebuild (pty-process-list-admission.ts:106-115).
|
||||
13. Detached shell is still re-adopted. Seed a leafId-less lease whose relay-reported `paneKey` names a DIFFERENT tab than the live pane: assert the shell is attached exactly once and no second shell spawns. Mutation: match on full-string paneKey. This is the migration gate — green before K, re-run after.
|
||||
14. Paired-client visibility survives the state rename. Record in `detached`, ptyId absent from every layout: assert `buildHeadlessMobileSessionTerminalTabs` still yields the tab. Mutation: drop the `expired`-equivalent — orca-runtime.ts:5964/:6290/:6322/:6359 must all be covered, not just the dead :16478 caller.
|
||||
15. Cleanup UI reaches the user's own abandoned shells. Delete the repo row (new `repoId` on re-add) while shells run: assert they are offered for cleanup. Mutation: reinstate worktreeId-based "owned elsewhere" — must redden.
|
||||
16. Nothing is killed implicitly. Any reattach/cleanup pass over foreign or unrecognized shells: assert zero kill RPCs. Mutation: auto-kill unclassified orphans.
|
||||
17. One pane, one live shell across partitions. Two live shells for one `(worktree, tab, leaf)` with the binding only in `local`: assert exactly one is bound. Mutation: restore the ssh-first preference in `durablyBoundPtyIdForPane` (persistence.ts:7271).
|
||||
18. P covers readers. Seed a binding only in `local`, call stable-pane resolve/retire with `connectionId` set: must find it. Mutation: revert either reader.
|
||||
19. Adopt is exclusive. Two concurrent adopts of one orphan: one succeeds, the other gets a typed refusal.
|
||||
20. Renderer cannot clobber ownership. Publish a stale full `WorkspaceSessionState` via `session:set`: assert the ownership map is byte-identical.
|
||||
21. Offline quit path. Restorable predicate with store only and a null mux: must not throw, must not require an RPC (persistence.ts:6465).
|
||||
22. Folder-workspace suffix. Two `::workspace:` instances on one path: a record for A must not reattach into B.
|
||||
23. Target reassign carries ownership. `reassignSshTargetId(old, new)`: every record's `hostId` field and the reverse index follow, with no re-keying.
|
||||
24. Windows/WSL relay kill. Assert the rules produce `detached`, never `terminated`, and that no UI copy asserts death. Mutation: assert-death — must redden on a Windows fixture even though it would pass on POSIX.
|
||||
25. Old-relay degradation. `listProcesses` without `paneKey`, grant without `relayInstanceId`: projection non-empty, nothing killed, reattach falls back to the `getPtyIdsForConnection UNION records` work list, and the per-attach incarnation guard still rejects a recycled id.
|
||||
|
||||
=====================================================================
|
||||
D12. RE-TEST AGAINST THE SECTION-3 REFUTATIONS
|
||||
=====================================================================
|
||||
REFUTATION A LEG 1 — "a lease must name a live shell no pane owns." SURVIVES. Discovery never was the lease's job: reattach builds its list from `getPtyIdsForConnection(target) UNION` the durable rows (ssh-relay-session.ts:2211-2219), and the connect-time projection adds host-reported shells on top. Leaf-keying does not weaken this. D5's RELEASE-vs-RETIRE distinction is REQUIRED BY this leg: a record whose leaf is gone must be released (shell stays discoverable and adoptable) rather than deleted (knowledge lost) or terminated (respawn wrongly authorized).
|
||||
|
||||
REFUTATION A LEG 2 — "a lease outlives the binding; expiry authorizes a 30s recreate." First clause survives exactly: `detached`/`terminated` are states on a main-owned, top-level record that persists after the layout binding is wiped. The recreate clause is PRESERVED AS A CONDITIONAL, not deleted: E-2 keeps a grant, gated on `relayInstanceId` equality, and only after E-1 shows the branch executes at production id shapes. Deleting the grant outright — proposed twice — is still refused as a simplification, because it dies to A leg 2 as written and rests on a read-not-run conclusion; it remains available as an E-1 OUTCOME. What does ship regardless is the user affordance, which covers every case a narrowed grant would drop.
|
||||
|
||||
REFUTATION B — "make the host authoritative." SURVIVES, and is honored more cheaply than before. Ownership stays on local disk, top-level, main-owned, read synchronously inside `setLocalWorkspaceSession` during quit with the host down (D8). Everything this design changes is the record's KEY and WHAT MAY TRANSITION IT — never where it lives or how it is read. The host is consulted only while connected, and only for "which shells are in your Map?", "is this the same shell I bound?" (incarnation) and "are you the same process?" (relayInstanceId) — three questions it answers from data it already holds. Critically, D4's deletion of retire-on-absence is what keeps refutation B intact under load: a design that retired durable ownership because a remote listing timed out would have made the host authoritative by accident.
|
||||
|
||||
REFUTATION C — a second authority architecture. Untouched, and this revision deletes more than the last: no decision table, no `hostEpoch`, no `os.uptime` math, no per-platform boot semantics, no paneKey transfer/alias layer, no attach-time identity apparatus, no orphan-ownership classifier, no second durable record.
|
||||
|
||||
WHERE THE CUT IS. Plane 1 (delivery identity, credit ledger, chunk-and-ack, binary payloads, loopback WebSocket) proceeds in parallel. The cut is DURABILITY SCOPE vs EVIDENCE SOURCE, and this revision now applies that rule to all four inferences the prior round left standing: an error string means dead (deleted, D2), absence from a list means dead (deleted, D4/D5), a delivery-checkpoint flag means alive (deleted, D3), an arithmetic clock means rebooted (never built, D3). What remains on the wire is two optional fields: `relayInstanceId` on the grant and `paneKey` on the process summary. RC3 is entirely Plane 1's.
|
||||
@@ -1,255 +0,0 @@
|
||||
# Terminal session correctness — design overview
|
||||
|
||||
Plain-language explanation of what breaks, why, and what we changed. No prior
|
||||
context needed.
|
||||
|
||||
---
|
||||
|
||||
## 1. The cast of characters
|
||||
|
||||
When you look at a terminal in Orca, five different things are lined up behind
|
||||
that one rectangle on screen.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph YOU["What you see"]
|
||||
PANE["Pane<br/><i>a rectangle in a tab</i>"]
|
||||
end
|
||||
subgraph APP["Orca's bookkeeping"]
|
||||
BIND["Binding<br/><i>this pane uses that shell</i>"]
|
||||
LEASE["Lease<br/><i>a note saying: we own<br/>a remote shell</i>"]
|
||||
end
|
||||
subgraph MACHINE["The actual computer"]
|
||||
PTY["PTY<br/><i>the shell process</i>"]
|
||||
HOST["Host<br/><i>your Mac, or an SSH box</i>"]
|
||||
end
|
||||
|
||||
PANE -->|bound to| BIND
|
||||
BIND -->|points at| PTY
|
||||
LEASE -->|claims| PTY
|
||||
PTY -->|runs on| HOST
|
||||
```
|
||||
|
||||
The pane is what you see. The **PTY** is the real shell process doing the work.
|
||||
The **binding** is the note that connects them. On a remote machine there is also
|
||||
a **lease** — a second note saying "we still own that remote shell."
|
||||
|
||||
**Everything that went wrong is one of those notes pointing at the wrong thing.**
|
||||
|
||||
---
|
||||
|
||||
## 2. The bug, in one picture
|
||||
|
||||
You are working on an SSH machine. Your network hiccups. Orca reconnects.
|
||||
|
||||
**What should happen:** your two terminals come back, same shells, same work.
|
||||
|
||||
**What actually happened:**
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["You have 2 terminals"] --> B["Network hiccup"]
|
||||
B --> C["Orca reconnects"]
|
||||
C --> D["Now you have<br/>19 terminals"]
|
||||
D --> E["Hiccup again"]
|
||||
E --> F["Now 20.<br/>Most are ghosts you<br/>never opened"]
|
||||
|
||||
style D fill:#ffe6e6,stroke:#cc0000
|
||||
style F fill:#ffcccc,stroke:#cc0000
|
||||
```
|
||||
|
||||
That is a real report: **2 → 19 → 20**. Meanwhile the remote machine fills up
|
||||
with shells nobody is using, until it refuses to start any more.
|
||||
|
||||
A second, worse version: your AI coding agent gets **resumed twice**. Two copies
|
||||
of `claude` writing into the same conversation file. One report had **five**.
|
||||
|
||||
---
|
||||
|
||||
## 3. Why it happened — three root causes
|
||||
|
||||
### Cause 1: the lease forgot which pane it belonged to
|
||||
|
||||
Each lease is a note saying "we own remote shell X." The note _had_ a field for
|
||||
"which pane," but nobody checked it.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph BEFORE["Before — lease identified by shell only"]
|
||||
P1["Pane A"] --> L1["Lease: shell-1"]
|
||||
P1 -.->|reconnect makes<br/>a new shell| L2["Lease: shell-2"]
|
||||
P1 -.->|and another| L3["Lease: shell-3"]
|
||||
N1["Nothing retires<br/>the old notes"]
|
||||
end
|
||||
style N1 fill:#ffe6e6,stroke:#cc0000
|
||||
```
|
||||
|
||||
Every reconnect added a note and removed none. Next reconnect, Orca dutifully
|
||||
restored **all** of them — one pane per stale note.
|
||||
|
||||
**Fix:** a lease is now identified by _which pane_ it belongs to, not just which
|
||||
shell. One pane, one live lease. Older notes are retired.
|
||||
|
||||
> Retired means **marked expired, not killed**. The remote shell is deliberately
|
||||
> left running — losing a note is not proof the shell died.
|
||||
|
||||
### Cause 2: reconnecting could _create_ panes
|
||||
|
||||
The function that says "pane A uses shell X" was also allowed to _invent_ pane A
|
||||
if it didn't exist. That is correct when you open a new terminal. It is very
|
||||
wrong when reconnecting.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
R["Reconnect finds a<br/>lease for shell-7"] --> Q{"Does its pane<br/>still exist?"}
|
||||
Q -->|"Yes"| OK["Reattach. Correct."]
|
||||
Q -->|"No — before"| BAD["Invent the pane.<br/>Ghost appears."]
|
||||
Q -->|"No — after"| GOOD["Refuse. Leave the shell<br/>running, unbound."]
|
||||
|
||||
style BAD fill:#ffcccc,stroke:#cc0000
|
||||
style GOOD fill:#e6ffe6,stroke:#00aa00
|
||||
```
|
||||
|
||||
**Fix:** reattach now says _"bind only, never create."_ Spawning a new terminal
|
||||
still creates, exactly as before.
|
||||
|
||||
### Cause 3: a living shell was reported as dead
|
||||
|
||||
The remote side has two different bad-news messages:
|
||||
|
||||
| Real meaning | What it was reported as |
|
||||
| ------------------------------------------------------------- | ------------------------------ |
|
||||
| "This shell is gone." | `SESSION_EXPIRED` ✅ correct |
|
||||
| "Shell is fine, I just need to re-establish the output pipe." | `SESSION_EXPIRED` ❌ **wrong** |
|
||||
|
||||
Orca's reaction to "expired" is to start a fresh shell — and because it passes
|
||||
the old session id along, the AI agent gets resumed a second time. **That is the
|
||||
duplicate-agent bug.**
|
||||
|
||||
**Fix:** the second case gets its own message, and respawning now requires
|
||||
_positive proof_ the shell is gone. Anything unrecognised means "unknown", and
|
||||
unknown never justifies starting a replacement.
|
||||
|
||||
---
|
||||
|
||||
## 4. The one rule underneath all of it
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
E["Evidence"] --> A{"What does it<br/>prove?"}
|
||||
A -->|"Shell exited"| D["DEAD — cleanup allowed"]
|
||||
A -->|"Can't reach it"| U["UNKNOWN"]
|
||||
A -->|"Timed out"| U
|
||||
A -->|"Not in a list"| U
|
||||
A -->|"Tried 3 times"| U
|
||||
U --> N["Do nothing destructive.<br/>Wait. Retry. Ask."]
|
||||
|
||||
style D fill:#e6ffe6,stroke:#00aa00
|
||||
style U fill:#fff4e6,stroke:#dd8800
|
||||
style N fill:#fff4e6,stroke:#dd8800
|
||||
```
|
||||
|
||||
**Unknown is not dead.** A disconnect, a timeout, an empty list, or three failed
|
||||
retries tell you nothing about whether a process is alive. Every bug above is
|
||||
some version of treating silence as a death certificate.
|
||||
|
||||
We also made the liveness check able to _say_ "unknown". It used to return only
|
||||
yes/no — so a provider whose network was down had no way to express "I can't
|
||||
see, don't ask me" and answered "no", meaning dead.
|
||||
|
||||
---
|
||||
|
||||
## 5. What we changed
|
||||
|
||||
Six changes, ~214 lines of production code total.
|
||||
|
||||
| # | Change | Stops |
|
||||
| --- | ----------------------------------------------- | ----------------------------------------------- |
|
||||
| 1 | Reattach binds, never creates | Ghost panes |
|
||||
| 2 | Leases identified by pane | 2 → 19 → 20 growth |
|
||||
| 3 | Heal already-corrupted lease data | Existing installs stuck at 20 |
|
||||
| 4 | "Needs re-establishing" ≠ "expired" | Duplicate agent resume |
|
||||
| 5 | Liveness can answer "unknown" | Live shells declared dead |
|
||||
| 6 | One PTY's failure can't drop the shared channel | One bad pane killing every session on that host |
|
||||
|
||||
### Change 6 in a picture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph B["Before"]
|
||||
X1["Pane 3 can't re-prove<br/>its output stream"] --> X2["Give up after N tries"]
|
||||
X2 --> X3["Drop the entire<br/>connection to that host"]
|
||||
X3 --> X4["Every pane, file transfer<br/>and git command dies"]
|
||||
end
|
||||
subgraph A["After"]
|
||||
Y1["Pane 3 can't re-prove<br/>its output stream"] --> Y2["Park pane 3 only"]
|
||||
Y2 --> Y3["Its shell keeps running.<br/>Siblings untouched."]
|
||||
end
|
||||
style X4 fill:#ffcccc,stroke:#cc0000
|
||||
style Y3 fill:#e6ffe6,stroke:#00aa00
|
||||
```
|
||||
|
||||
### And a guard on every keystroke
|
||||
|
||||
Orca queues your typing. If a pane switches to a different shell mid-flight,
|
||||
queued keystrokes could land on the **new** shell.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
K["Keystroke for shell-1"] --> C{"Is shell-1 still<br/>this pane's shell?"}
|
||||
C -->|"Yes"| S["Deliver"]
|
||||
C -->|"No — pane moved on"| R["Refuse"]
|
||||
C -->|"Pane unknown"| S2["Deliver<br/><i>unknown ≠ stale</i>"]
|
||||
|
||||
style R fill:#fff4e6,stroke:#dd8800
|
||||
```
|
||||
|
||||
We did this **without changing the wire format** — Orca already keeps two lookup
|
||||
tables in lock-step, so if they disagree, that _is_ proof the id is stale. No new
|
||||
data is sent per keystroke.
|
||||
|
||||
---
|
||||
|
||||
## 6. The approach we rejected
|
||||
|
||||
A previous attempt built a **new authority architecture beside the old one**:
|
||||
+60,903 lines of production code. It fixed **none** of the three root causes —
|
||||
they were still present, untouched, underneath.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph W["Rejected"]
|
||||
O1["Old system<br/>(3 bugs)"]
|
||||
N1["New system<br/>+60,903 lines"]
|
||||
O1 -.->|"bugs still here"| O1
|
||||
end
|
||||
subgraph R["Taken"]
|
||||
O2["Old system<br/>(3 bugs fixed)<br/>+214 lines"]
|
||||
end
|
||||
style N1 fill:#ffcccc,stroke:#cc0000
|
||||
style O2 fill:#e6ffe6,stroke:#00aa00
|
||||
```
|
||||
|
||||
A second proposal of mine was also rejected in review — it would have added a
|
||||
_second_ identity system beside an existing one, which is the same mistake in
|
||||
miniature. Three of its factual claims turned out to be false when checked
|
||||
against the code.
|
||||
|
||||
**The lesson both times:** the machinery mostly existed already. The bug was that
|
||||
it wasn't being _called_.
|
||||
|
||||
---
|
||||
|
||||
## 7. What is still open
|
||||
|
||||
| Thing | State |
|
||||
| ------------------------ | -------------------------------------------------------------------------------- |
|
||||
| The three root causes | Fixed |
|
||||
| Duplicate agent resume | Root cause fixed; the codebase still carries 3 older patches for it |
|
||||
| Orphaned remote shells | Left running on purpose. Not reachable from the UI yet — needs a recovery screen |
|
||||
| The keystroke quarantine | Still needed. We proved deleting it re-opens a real `rm -rf` hazard |
|
||||
|
||||
**On that last one:** there is a safety module that drops the tail of a half-typed
|
||||
line after a recovery, because `echo hi; rm -rf x` can otherwise arrive as
|
||||
`cho hi; rm -rf x` — the shell rejects `cho` and then happily runs `rm -rf x`. We
|
||||
tried to delete it. We proved by experiment that we cannot, yet.
|
||||
File diff suppressed because one or more lines are too long
-324
@@ -1,324 +0,0 @@
|
||||
query TerminalSessionCorrectnessRelatedWork {
|
||||
repository(owner: "stablyai", name: "orca") {
|
||||
n8129: issueOrPullRequest(number: 8129) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n8275: issueOrPullRequest(number: 8275) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n8585: issueOrPullRequest(number: 8585) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n8618: issueOrPullRequest(number: 8618) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9034: issueOrPullRequest(number: 9034) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9092: issueOrPullRequest(number: 9092) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9093: issueOrPullRequest(number: 9093) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9138: issueOrPullRequest(number: 9138) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9562: issueOrPullRequest(number: 9562) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9585: issueOrPullRequest(number: 9585) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9819: issueOrPullRequest(number: 9819) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9820: issueOrPullRequest(number: 9820) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9827: issueOrPullRequest(number: 9827) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n9833: issueOrPullRequest(number: 9833) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n10118: issueOrPullRequest(number: 10118) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n10208: issueOrPullRequest(number: 10208) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n10235: issueOrPullRequest(number: 10235) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n10385: issueOrPullRequest(number: 10385) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n10415: issueOrPullRequest(number: 10415) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n10744: issueOrPullRequest(number: 10744) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11006: issueOrPullRequest(number: 11006) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11265: issueOrPullRequest(number: 11265) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11339: issueOrPullRequest(number: 11339) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11342: issueOrPullRequest(number: 11342) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11495: issueOrPullRequest(number: 11495) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11574: issueOrPullRequest(number: 11574) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11575: issueOrPullRequest(number: 11575) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11596: issueOrPullRequest(number: 11596) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11622: issueOrPullRequest(number: 11622) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11729: issueOrPullRequest(number: 11729) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11800: issueOrPullRequest(number: 11800) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11801: issueOrPullRequest(number: 11801) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11803: issueOrPullRequest(number: 11803) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11804: issueOrPullRequest(number: 11804) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n11904: issueOrPullRequest(number: 11904) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12036: issueOrPullRequest(number: 12036) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12134: issueOrPullRequest(number: 12134) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12140: issueOrPullRequest(number: 12140) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12220: issueOrPullRequest(number: 12220) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12241: issueOrPullRequest(number: 12241) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12264: issueOrPullRequest(number: 12264) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12339: issueOrPullRequest(number: 12339) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12447: issueOrPullRequest(number: 12447) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12448: issueOrPullRequest(number: 12448) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12456: issueOrPullRequest(number: 12456) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12474: issueOrPullRequest(number: 12474) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12477: issueOrPullRequest(number: 12477) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12568: issueOrPullRequest(number: 12568) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12600: issueOrPullRequest(number: 12600) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12634: issueOrPullRequest(number: 12634) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12683: issueOrPullRequest(number: 12683) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12699: issueOrPullRequest(number: 12699) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12702: issueOrPullRequest(number: 12702) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12743: issueOrPullRequest(number: 12743) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12749: issueOrPullRequest(number: 12749) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12751: issueOrPullRequest(number: 12751) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12760: issueOrPullRequest(number: 12760) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12768: issueOrPullRequest(number: 12768) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12798: issueOrPullRequest(number: 12798) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12882: issueOrPullRequest(number: 12882) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12903: issueOrPullRequest(number: 12903) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n12987: issueOrPullRequest(number: 12987) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n13013: issueOrPullRequest(number: 13013) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n13110: issueOrPullRequest(number: 13110) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
n13111: issueOrPullRequest(number: 13111) {
|
||||
...IssueDetails
|
||||
...PullRequestDetails
|
||||
}
|
||||
p13110: pullRequest(number: 13110) {
|
||||
...ReviewDetails
|
||||
}
|
||||
p13111: pullRequest(number: 13111) {
|
||||
...ReviewDetails
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fragment IssueDetails on Issue {
|
||||
number
|
||||
title
|
||||
url
|
||||
state
|
||||
stateReason
|
||||
updatedAt
|
||||
}
|
||||
|
||||
fragment PullRequestDetails on PullRequest {
|
||||
number
|
||||
title
|
||||
url
|
||||
state
|
||||
isDraft
|
||||
merged
|
||||
reviewDecision
|
||||
mergeable
|
||||
mergeStateStatus
|
||||
headRefName
|
||||
baseRefName
|
||||
additions
|
||||
deletions
|
||||
changedFiles
|
||||
updatedAt
|
||||
commits(last: 1) {
|
||||
nodes {
|
||||
commit {
|
||||
oid
|
||||
statusCheckRollup {
|
||||
state
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fragment ReviewDetails on PullRequest {
|
||||
reviewThreads(first: 100) {
|
||||
nodes {
|
||||
isResolved
|
||||
isOutdated
|
||||
comments(last: 1) {
|
||||
nodes {
|
||||
url
|
||||
author {
|
||||
login
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,683 +0,0 @@
|
||||
# Goalposts and proof contract
|
||||
|
||||
This file answers one question: **what exact evidence is required before an
|
||||
agent may write “proven”?**
|
||||
|
||||
The current status is **0/8 proven goalposts** and **2/13 proven
|
||||
journeys**. Status may change only from evidence produced on the same rebased,
|
||||
converged release candidate. Historical snapshot tests and independently useful
|
||||
narrow PRs may be cited as partial evidence but cannot promote a row.
|
||||
|
||||
## Recorded user decisions
|
||||
|
||||
These amend the contract above. Only the user may add to this list.
|
||||
|
||||
### D1 — G6 relaxed from strictly net-negative (2026-08-07)
|
||||
|
||||
> "It's okay if we must increase LOC but try not to."
|
||||
|
||||
G6's pass condition is no longer a strict inequality against the frozen
|
||||
baseline. It is now: **minimise added production code, and justify every net
|
||||
addition against the correctness it buys.** A net-positive total does not fail
|
||||
G6 on its own; an unjustified one does.
|
||||
|
||||
Why this was needed: the deletion budget the plan assumed does not exist. An
|
||||
entrypoint-rooted import graph over all 20 real build entrypoints found that
|
||||
51 of the 53 candidate files in `src/relay/*pty-source-*`,
|
||||
`src/shared/pty-consumer-*`, and `src/main/ipc/ssh-pty-*` are reachable and
|
||||
value-instantiated on live paths. Only 2 files (263 LOC) are unreachable, and 2
|
||||
of the candidates did not exist at the baseline so deleting them earns no
|
||||
credit. Against roughly +1,021 LOC to offset, a strict inequality was not
|
||||
reachable without deleting load-bearing code — which the governing rule forbids.
|
||||
|
||||
Still binding: correctness may not be weakened to reduce line count, and a
|
||||
replacement architecture added beside the old one does not earn its lines.
|
||||
|
||||
## Proven journeys
|
||||
|
||||
### Journey evidence not yet sufficient to promote
|
||||
|
||||
**Journey 2 (daemon half).** `tests/e2e/daemon-restart-session-liveness.spec.ts`,
|
||||
3 tests. Two mutations each redden exactly one clause, on macOS and Linux:
|
||||
reverting three-valued `hasPty` reddens only the unknown-not-dead clause;
|
||||
widening the sole-provider fallback reddens only the stale-generation clause.
|
||||
The lead re-verified the first mutation independently. **Missing: physical WSL**,
|
||||
which the journey names explicitly. That host is now unblocked (the distro has a
|
||||
provisioned default user) and the run is outstanding.
|
||||
|
||||
**Journey 12.** `tests/e2e/cross-version-wire/`, 13 tests against a real published
|
||||
baseline. Reverting the restore-required publication to expiry reddens 7 of 13;
|
||||
restore greens. Lead-verified. **Missing: live skew.** These are in-process wire
|
||||
tests. The original ledger named live paired-runtime and SSH skew as the gap, and
|
||||
an in-process decoder test does not close it.
|
||||
|
||||
**Journey 4.** Sibling-pane isolation on one host reddens when
|
||||
`mux.dispose('connection_lost')` is restored. **The cross-host clause cannot be
|
||||
proven by mutation**: a mux belongs to one relay session per SSH target, so its
|
||||
dispose cannot cross a host boundary — the cross-host test stayed green under the
|
||||
mutation, which is the empirical receipt. That clause rests on
|
||||
isolation-by-construction, and saying otherwise would be a false claim.
|
||||
|
||||
**Journey 6.** `tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts`,
|
||||
4 tests against a real Docker OpenSSH container configured `MaxSessions 1`, the
|
||||
cap read back from `sshd -T` inside each test. Three of the four clauses
|
||||
discriminate, clause-selectively, in single runs (the spec is not `serial`, so a
|
||||
red test never reports the others as "did not run"):
|
||||
|
||||
- dropping `mayCreate: false` from the reattach binding write reddens only the
|
||||
authority-reconnect test, with the unowned leaf grafted into the `local`
|
||||
partition;
|
||||
- marking leases `terminated` instead of `detached` in `beginShutdownDetach`
|
||||
reddens only the two restart tests, with a cold-spawned shell beside the
|
||||
surviving one.
|
||||
|
||||
**Missing: the transport-disconnect clause does not discriminate.** Four guard
|
||||
removals were tried and it stayed green under all of them, including publishing
|
||||
`SSH_SESSION_EXPIRED` in place of `SSH_SOURCE_RESTORE_REQUIRED` and making
|
||||
`isProvenSshSessionGoneError` return true for every error. A sever that leaves
|
||||
the detached relay holding its delivery record reattaches through the checkpoint
|
||||
path, so no reattach failure is ever classified and the respawn-requires-proof
|
||||
guards are off that path. Inducing `restoreRequired` needs a lost or mismatched
|
||||
delivery record, which this fault does not produce. That clause is therefore a
|
||||
forward guard and cannot promote the row. Also missing: only Docker-on-macOS was
|
||||
run; the explicit-close/proven-teardown retirement clause is not covered.
|
||||
|
||||
**Journey 13.** Not proven. One of ten named dimensions measured. The
|
||||
`isSupersededPtyId` fence costs roughly 14ns per call, but it was measured on
|
||||
lifted predicates in plain Node, not through real Electron IPC — that part is an
|
||||
inference, not a measurement.
|
||||
|
||||
### Correction: Journey 12's cross-version tests exercise a route production does not have
|
||||
|
||||
Recorded earlier that the cross-version suite "confirms the new
|
||||
`SSH_SOURCE_RESTORE_REQUIRED` token mutates nothing on an old client." That
|
||||
claim is withdrawn.
|
||||
|
||||
An attempt to build a LIVE two-process skew oracle established that the token
|
||||
never crosses a version boundary at all. It is minted in main
|
||||
(`src/main/providers/ssh-pty-provider.ts:110`) and consumed in the same app's
|
||||
renderer (`pty-connection.ts:8833`, `:9080`) over Electron IPC — always one
|
||||
version. The only cross-version terminal boundary, paired client to HUB runtime,
|
||||
goes through `remote-runtime-pty-transport.ts`, whose sole respawn trigger is
|
||||
`SSH_SESSION_EXPIRED` (line 1481), and whose reattach never reaches
|
||||
`SshPtyProvider.spawn`.
|
||||
|
||||
The in-process suite injects the token into `terminal.resolvePane`, which is a
|
||||
pure lookup and not a route production takes. So those 13 tests are a decoder
|
||||
guard, not evidence about old clients.
|
||||
|
||||
Two consequences, both good: the wire-compatibility risk this branch was thought
|
||||
to introduce **does not exist**, because there is no version boundary for the new
|
||||
token to cross. And Journey 12 needs a different oracle entirely — one aimed at a
|
||||
boundary that is genuinely cross-version.
|
||||
|
||||
### Journey 6 — evidence, and why it is not promotable
|
||||
|
||||
`tests/e2e/ssh-maxsessions-remote-pid-binding-identity.spec.ts`, 4 tests, each
|
||||
with its own container and Electron profile, against real OpenSSH with
|
||||
`MaxSessions 1` read back from `sshd -T` rather than assumed. Remote pids are
|
||||
read on the container two independent ways and must agree, each carrying its
|
||||
kernel start time.
|
||||
|
||||
Two mutations discriminate and are disjoint: removing `mayCreate: false` reddens
|
||||
only the reconnect clause; marking leases `terminated` instead of `detached` at
|
||||
quit reddens only the two restart clauses.
|
||||
|
||||
**Not promotable:** the "same remote PID and exact binding survive a transport
|
||||
disconnect" clause is a forward guard. Four separate guard removals left it
|
||||
green. It asserts a real property, but nothing shipped is load-bearing for it,
|
||||
so it cannot be claimed as proof of this branch's work.
|
||||
|
||||
### Journey 3 — evidence, and why it is not promotable
|
||||
|
||||
`tests/e2e/ssh-lazy-discovery-skipped-host-restart.spec.ts`, 3 tests, two real
|
||||
containers, sampling sshd's own accept log and live session census 20+ times over
|
||||
22 seconds with the in-use host as a positive control.
|
||||
|
||||
**Not promotable:** no mutation reddens clause 3 alone. The genuine cross-host
|
||||
lease-scoping guard is load-bearing, but removing it breaks the sibling host
|
||||
during setup, so the failure carries no clause information.
|
||||
|
||||
### Journey 2 — Daemon and physical WSL (proven 2026-08-08)
|
||||
|
||||
Oracle: `tests/e2e/daemon-restart-session-liveness.spec.ts`, 3 tests, plus
|
||||
`tests/e2e/helpers/daemon-shell-process-identity.ts`. The PTY leader is a real
|
||||
login shell that reports `$$` back through the production write path
|
||||
(`DaemonPtyRouter.write` -> daemon socket -> PTY), and that pid is resolved to a
|
||||
kernel start time, so a look-alike respawn cannot pass as a survivor.
|
||||
|
||||
It replaced a spec that never crossed the daemon boundary and whose successor
|
||||
generation owned nothing, which made "the live successor is neither killed nor
|
||||
replaced" vacuous.
|
||||
|
||||
| Environment | Result | Discrimination watched |
|
||||
| -------------------------------------------- | -------- | --------------------------------------------------- |
|
||||
| macOS 26.3.1 arm64 | 3 passed | 2 mutations, each reddening one clause |
|
||||
| Ubuntu 24.04 x86_64 native | 3 passed | same 2 mutations, same single-clause reds |
|
||||
| WSL2 Ubuntu 26.04 on a physical Windows host | 3 passed | reverting three-valued `hasPty` reddens test 1 only |
|
||||
|
||||
Clause-selective on all three. The spec runs `mode: 'serial'`, so a red test 1
|
||||
reports 2 and 3 as "did not run" rather than passing — selectivity was therefore
|
||||
established by re-running 2 and 3 _alone under the same mutation_ and watching
|
||||
them stay green, not by assuming it.
|
||||
|
||||
Mutation A (`hasPty` reverted to `activeSessionIds.has(id)`) reddens only the
|
||||
unknown-not-dead clause. Mutation B (widening the sole-provider fallback in
|
||||
`daemon-session-owner-resolution.ts`) reddens only the stale-generation clause.
|
||||
The lead independently reproduced Mutation A on macOS.
|
||||
|
||||
Also confirmed on the WSL host, independently of the spec: an Orca WSL-mode
|
||||
terminal now starts and returns real output. It could not start before — the
|
||||
distro had no provisioned default Unix user, so every interactive `wsl.exe`
|
||||
launch blocked on first-run provisioning.
|
||||
|
||||
Limits, stated rather than implied:
|
||||
|
||||
- The WSL run used `ELECTRON_RUN_AS_NODE=1`, which is what this oracle needs
|
||||
(real daemon processes and real PTYs, no renderer). It does **not** show that
|
||||
Electron's GUI starts under WSL; no display server was started and that
|
||||
remains an open question for any journey needing a window.
|
||||
- WSL evidence is against `d74f5ed0eae` and on Node 22 rather than the required 24. The other two platforms ran the same oracle on Node 24.
|
||||
- `src/main/providers` is red on that distro for an unrelated reason:
|
||||
`local-pty-shell-ready.test.ts` pins an exact OSC 133 marker count and saw 7
|
||||
splits where it expects 4. The WSL run attributed that to bash 5.3.9, but that
|
||||
is **not** the cause — macOS runs the same bash 5.3.9 and the spec passes there
|
||||
67/67. The trigger is environmental to that distro, most plausibly a
|
||||
system-wide bashrc contributing prompt hooks, and the underlying defect is that
|
||||
the spec asserts an absolute count of markers it does not own. Outside this
|
||||
journey's surface, so left for its owner rather than reinterpreted blind — but
|
||||
it means "the unit suites are green on WSL" would be false.
|
||||
|
||||
### Journey 1 — Local macOS, Linux, and Windows (proven 2026-08-08)
|
||||
|
||||
Oracle: `tests/e2e/local-terminal-restart-binding-identity.spec.ts`. Two tests —
|
||||
the same pane, full binding and OS shell process survive renderer reload and app
|
||||
restart; and a stale pre-spawn session write is rejected instead of retiring the
|
||||
live binding.
|
||||
|
||||
Process identity is proved by the shell reporting its own pid through the
|
||||
production write path, plus the kernel-reported start time, so a recycled pid
|
||||
cannot pass as a survivor.
|
||||
|
||||
| Platform | Result | Discrimination watched |
|
||||
| --------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| macOS 26.3.1 arm64 | 2 passed | mutation reddens; restore greens |
|
||||
| Ubuntu 24.04 x86_64, native | 2 passed | 4 cycles, incl. deleting `restorableBindings` reconciliation — reddens test 2 only |
|
||||
| Windows 11 26200, native | 2 passed | A: forcing `shutdownDaemon()` in will-quit reddens test 1 (pid 13756 -> 8852, start times 5.4s apart). B: disabling the same reconciliation reddens test 2 only |
|
||||
|
||||
Every run used an isolated TMPDIR, because the harness keys its seeded-repo
|
||||
pointer on a machine-global tmpdir path and concurrent runs can both fabricate
|
||||
and mask a red.
|
||||
|
||||
Selectivity holds on two platforms: mutation B reddens only the stale-operation
|
||||
test and leaves the restart test green, so the two clauses are independently
|
||||
proved rather than jointly.
|
||||
|
||||
Residual limit, stated: "every stale exact operation" is proved for the
|
||||
pre-spawn session-write class. Local `pty:write`/`resize`/`signal` are fenced by
|
||||
`isSupersededPtyId` with its own oracle, but that fence compares a binding, not
|
||||
an incarnation — see `binding-identity-design.md`.
|
||||
|
||||
## Universal proof rule
|
||||
|
||||
A gate is `proven` only when both exist:
|
||||
|
||||
1. the complete behavior is reachable through every named production path; and
|
||||
2. the named proof exercises that behavior and its failure boundaries on the
|
||||
final candidate.
|
||||
|
||||
Every proof receipt must record:
|
||||
|
||||
- candidate commit SHA and exact merge base;
|
||||
- unfixed baseline SHA used for red/green discrimination;
|
||||
- exact command or manual journey protocol;
|
||||
- OS, architecture, filesystem, runtime, Git, SSH, daemon, relay, client, and
|
||||
host versions relevant to that journey;
|
||||
- UTC start/end time, exit status, counts, and artifact/log location;
|
||||
- the production caller, persistence boundary, transport, and cleanup path
|
||||
exercised;
|
||||
- the expected oracle and the observed result; and
|
||||
- independent reviewer identity and unresolved-finding count.
|
||||
|
||||
For a new regression oracle, demonstrate that it fails for the intended reason
|
||||
on the unfixed baseline and passes on the candidate. A test that passes both is
|
||||
a forward guard and cannot prove the fix.
|
||||
|
||||
## G0 — One design contract
|
||||
|
||||
**Current status: partial.**
|
||||
|
||||
G0 is proven only when:
|
||||
|
||||
- every reachable production path follows one reconciled identity, host
|
||||
boundary, operations, delivery, migration, compatibility, and minimal-shape
|
||||
contract;
|
||||
- there is one final-host authority service, thin transport adapters, one app
|
||||
projection/controller, and one bounded pre-cutover legacy importer;
|
||||
- no adapter or renderer owns a parallel authority state machine;
|
||||
- all retained existing primitives are mapped by semantics, not similar names;
|
||||
- the lost-consumer/compaction liveness gap has a safe, bounded, authenticated,
|
||||
operator-reachable resolution that does not infer death from time; and
|
||||
- an independent production-graph audit finds no contradictory path.
|
||||
|
||||
The design document alone, a behavior contract, or focused tests do not prove
|
||||
G0.
|
||||
|
||||
## G1 — One final-host authority
|
||||
|
||||
**Current status: partial.**
|
||||
|
||||
G1 is proven only when local, daemon, WSL, direct SSH, nested SSH, paired
|
||||
runtime, and remote server all resolve:
|
||||
|
||||
- a stable identity minted or validated by the final PTY-owning host;
|
||||
- a canonical host-local namespace for worktree, folder, and floating
|
||||
workspaces;
|
||||
- one exact pane-generation/PTY-incarnation binding;
|
||||
- host connections keyed by final-host identity with lazy discovery;
|
||||
- concurrent host isolation; and
|
||||
- namespace-local admission, failure, grant, handover, and retirement.
|
||||
|
||||
`connectionId`, SSH target ID, client repository ID, path spelling, or
|
||||
`worktreeId` may be routing metadata. Their existing names and usage counts are
|
||||
not proof that they satisfy final-host identity or namespace semantics.
|
||||
|
||||
Proof requires the applicable live journeys below, including simultaneous hosts
|
||||
and independent client/host updates.
|
||||
|
||||
## G2 — Exact operations only
|
||||
|
||||
**Current status: partial.**
|
||||
|
||||
G2 is proven only when input, resize, signal, close, output, and exit are fenced
|
||||
by the full binding captured before any await:
|
||||
|
||||
- authority host;
|
||||
- namespace;
|
||||
- pane generation;
|
||||
- owner/writer incarnation;
|
||||
- physical PTY;
|
||||
- PTY incarnation; and
|
||||
- negotiated operation/source generation where applicable.
|
||||
|
||||
Stale, partial, absent, timed-out, disconnected, or unknown evidence must not
|
||||
affect a successor. An authoritative operation must never retry through an
|
||||
ID-only provider call or a legacy mutation path.
|
||||
|
||||
Proof must cover all operations across local, daemon, WSL, SSH, paired runtime,
|
||||
remote server, renderer fallback, restart, concurrent replacement, and both
|
||||
mixed-version directions. Store-row counts and source-text assertions are not
|
||||
sufficient.
|
||||
|
||||
## G3 — Durable ordered delivery
|
||||
|
||||
**Current status: partial.**
|
||||
|
||||
G3 is proven only when the final production design provides all observable
|
||||
properties below, even if its internal mechanism differs from the preserved
|
||||
construction design:
|
||||
|
||||
- complete boundary snapshot before later events;
|
||||
- producer held while boundary/replay is established;
|
||||
- contiguous replay before reconciliation and live resume;
|
||||
- durable semantic outcomes, including exit and state needed by a newly
|
||||
attached consumer;
|
||||
- durable idempotent main-process projection before acknowledgement;
|
||||
- final-host-owned cumulative acknowledgement or an explicitly approved
|
||||
equivalent with the same crash/replay guarantees;
|
||||
- renderer snapshot-plus-delta observation;
|
||||
- app, renderer, host, and transport restart resume;
|
||||
- gap detection and resnapshot without silent omission;
|
||||
- bounded memory, queues, pages, listeners, timers, and retained output;
|
||||
- independent consumer retirement and safe compaction liveness; and
|
||||
- no app-side duplicate cursor, settlement, receipt, or suffix-reconciliation
|
||||
authority.
|
||||
|
||||
Proof must include crash-before/after-ACK cuts, lost responses, disconnected
|
||||
replay, gap recovery, slow/stalled consumers, retired and permanently lost
|
||||
consumers, paired/remote restart, mixed versions, and scale.
|
||||
|
||||
## G4 — One-way legacy cutover
|
||||
|
||||
**Current status: partial.**
|
||||
|
||||
G4 is proven only when each namespace performs, in order:
|
||||
|
||||
1. explicit capability negotiation;
|
||||
2. a brief legacy-write freeze;
|
||||
3. exact non-mutating inventory;
|
||||
4. a deterministic import plan;
|
||||
5. validation with ambiguity kept visible and non-destructive;
|
||||
6. one self-contained durable authority commit;
|
||||
7. topology attachment; and
|
||||
8. exact client opening through the authoritative path.
|
||||
|
||||
There must be no dual writer, destructive inference, authority-to-legacy
|
||||
fallback, or second durable migration catalog after cutover. Old peers remain
|
||||
on an unchanged isolated legacy surface or fail before mutation.
|
||||
|
||||
Proof requires crash cuts at every phase, replay from the self-contained commit,
|
||||
ambiguous-row isolation, independent namespace failure, and legacy-writer and
|
||||
reconciliation deletion.
|
||||
|
||||
## G5 — Wire and platform compatibility
|
||||
|
||||
**Current status: not started.**
|
||||
|
||||
G5 is proven only when all exchanged changes follow the remote-wire
|
||||
compatibility contract and the final candidate passes:
|
||||
|
||||
- old client to new host;
|
||||
- new client to old host;
|
||||
- native macOS;
|
||||
- native Linux at Ubuntu 20.04 / glibc 2.31 compatibility floor;
|
||||
- native Windows;
|
||||
- physical WSL, including Git Bash/`.cmd` boundaries where relevant;
|
||||
- Docker OpenSSH;
|
||||
- daemon;
|
||||
- direct and nested SSH;
|
||||
- paired runtime;
|
||||
- remote server;
|
||||
- git worktree;
|
||||
- folder workspace;
|
||||
- floating workspace; and
|
||||
- drive-letter and UNC namespace paths.
|
||||
|
||||
Run both skew directions independently across every changed deployment boundary:
|
||||
app↔daemon, app↔SSH relay/final host, paired client↔paired runtime, remote
|
||||
client↔remote server, and mobile/E2EE RPC where affected. A single in-process
|
||||
codec test or one client/host pairing cannot stand in for this matrix.
|
||||
|
||||
New opcodes or semantics require explicit capability negotiation. An optional
|
||||
field that parses on an old peer does not by itself prove that old behavior
|
||||
remains correct.
|
||||
|
||||
Mocking `process.platform`, running Linux inside Docker, or passing in-process
|
||||
wire unit tests does not prove the corresponding native or live-skew row.
|
||||
|
||||
## G6 — Simpler, minimised production code
|
||||
|
||||
**Current status: partial; structural clauses remain open.**
|
||||
|
||||
Decision D1 replaces the original strict inequality: the final integrated
|
||||
program must minimise production source and justify every net addition against
|
||||
the correctness it buys.
|
||||
|
||||
The default baseline is
|
||||
`5ed45739e94bdf6460364e033bfcec9b32c0b42a`, the base recorded by GitHub for
|
||||
PR #12600. This broader program subsumes #12600. Changing the baseline requires
|
||||
an explicit user decision recorded before more implementation begins.
|
||||
|
||||
G6 is proven only when:
|
||||
|
||||
- aggregate program-attributable production source net LOC is **minimised and
|
||||
every net addition justified** against the frozen baseline — see decision D1
|
||||
under "Recorded user decisions", which replaced the original strict
|
||||
inequality once the assumed deletion budget was shown not to exist;
|
||||
- every program-attributable prerequisite merged after the baseline and every
|
||||
stacked PR is included, even if a later rebase places it in `main`;
|
||||
- overlapping changes are recomputed from the frozen baseline to the final tree
|
||||
so additions and later deletions are not double-counted;
|
||||
- unrelated upstream or user deletions cannot offset program additions;
|
||||
- production, test, documentation, CI/runner, generated, and vendored changes
|
||||
are reported separately;
|
||||
- every new production module is reachable from a real entrypoint;
|
||||
- no test fixture remains under production compilation;
|
||||
- there is one identity comparison, transition implementation, exact-operation
|
||||
client, mutation admission path, and delivery state machine;
|
||||
- no re-export shim or one-type module exists solely to preserve construction
|
||||
layering;
|
||||
- no superseded quarantine, sliding-window, retry-verdict, reconciliation,
|
||||
duplicate cursor, receipt ledger, legacy writer, or migration bridge remains
|
||||
reachable after cutover; and
|
||||
- an independent reachability and duplicate-state-machine audit is clean.
|
||||
|
||||
Tests or docs cannot offset positive production LOC. A smaller narrow PR is not
|
||||
proof if the final stack remains net positive. Deleting correctness or platform
|
||||
coverage to hit the number is prohibited.
|
||||
|
||||
Classify by behavior, not directory name: shipped runtime code, migration code,
|
||||
and build-time code that enforces a shipped artifact invariant are production;
|
||||
test-only runners and fixtures are tests even when misplaced, and their presence
|
||||
under production compilation independently fails this gate. Publish the final
|
||||
file-by-file classification so the count cannot be moved between buckets.
|
||||
|
||||
### G6 clause assessment (2026-08-08)
|
||||
|
||||
Checked against the current branch rather than assumed:
|
||||
|
||||
| Clause | State |
|
||||
| ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| aggregate net production LOC minimised and justified (D1) | +228, each addition justified in its commit; 201 LOC deleted after verifying unreachability from all 20 build entrypoints |
|
||||
| every new production module reachable from a real entrypoint | holds — no new unreachable module was added |
|
||||
| no test fixture under production compilation | not audited on this branch |
|
||||
| one identity comparison / transition / exact-op client / admission path / delivery state machine | **fails** — several remain, and this is the clause the rejected binding proposal would have made worse by adding a second identity comparison beside `resolveStablePaneOwner` |
|
||||
| no re-export shim or one-type module for layering | holds for this branch's additions |
|
||||
| **no superseded quarantine, sliding-window, retry-verdict, reconciliation, duplicate cursor, receipt ledger, legacy writer, or migration bridge reachable** | **fails** — `terminal-input-quarantine.ts` is imported by `pty-connection.ts:98` and `terminal-pane-recovery.ts:6` |
|
||||
| independent reachability and duplicate-state-machine audit clean | not run |
|
||||
|
||||
On the quarantine specifically: it is **not** subsumed by the superseded-PTY
|
||||
fence, and claiming otherwise would be wrong. The fence refuses writes aimed at
|
||||
a stale ptyId. The quarantine guards the user's _subsequent_ typing — the tail
|
||||
of a half-typed line — landing on the successor shell under its current and
|
||||
correct ptyId, which the fence never sees.
|
||||
|
||||
Deleting it therefore requires contract property B: a _different_ shell surfaces
|
||||
unresolved and never silently receives the pane, at which point there is no
|
||||
mangled tail to suppress. That is a behaviour change to the recovery path, not a
|
||||
deletion, and an earlier attempt to shortcut it was rejected for reintroducing
|
||||
the `echo hi; rm -rf x` hazard by disarming per-pane state that is keyed per-tab.
|
||||
|
||||
G6 is therefore not promotable, and its blocking clauses are structural rather
|
||||
than arithmetic.
|
||||
|
||||
### The input quarantine is load-bearing, not superseded (2026-08-08)
|
||||
|
||||
G6's clause lists "no superseded quarantine ... remains reachable" and
|
||||
`terminal-input-quarantine.ts` was assumed to be one. It is not, and the
|
||||
evidence is direct.
|
||||
|
||||
**The hazard is live.** Disabling the single call site in `pty-connection.ts`
|
||||
and running the pane's own oracle reproduces it verbatim, lead-verified:
|
||||
|
||||
AssertionError: expected "vi.fn()" to not be called with arguments:
|
||||
[ 'cho hi; rm -rf x' ]
|
||||
|
||||
Deleting the module without a replacement re-opens `rm -rf` execution.
|
||||
|
||||
**Contract property B was costed by building it, not estimated.** Threading the
|
||||
incarnation to the renderer compiles at **+26 production LOC**, and completing it
|
||||
(publishing an incarnation from `resolveTerminalPane`, plus a new exported reader
|
||||
for the module-private `ptyIncarnationById` map) is about **+33**. The
|
||||
cross-remount state the comparison needs must outlive the destroyed pane, so it
|
||||
becomes a module of roughly the size of the one being deleted — the two in-tree
|
||||
precedents are 88 and 86 lines. Adding a renderer-side identity comparison would
|
||||
also worsen G6's already-failing "one identity comparison" clause. Floor: about
|
||||
**+140 production LOC to delete 88**.
|
||||
|
||||
**And the route is not uniformly available.** `RuntimeTerminalCreate` and
|
||||
`RuntimeTerminalResolvePane` carry no `incarnationId`; remote connect results are
|
||||
built without one; and the transport latches `resolvePaneUnavailable` when a host
|
||||
answers `method_not_found`. Mixed client/host versions are the normal state, and
|
||||
an optional field does not make an old host publish it. So a paired client reads
|
||||
_unknown_ — which under this program's own governing rule is not proof of "same
|
||||
shell". Requiring proof makes every remote reattach surface unresolved; not
|
||||
requiring it needs a fallback, and the only correct fallback is this quarantine.
|
||||
Either way the module stays reachable.
|
||||
|
||||
`endpointReplaced` is also routine rather than rare — a daemon death remounts
|
||||
every live pane — so property B would convert each into a manual per-pane
|
||||
reconnect, which is the coverage deletion G6 explicitly prohibits.
|
||||
|
||||
**Consequence:** this clause of G6 cannot be closed by deletion. Either the
|
||||
clause is amended to recognise the module as load-bearing, or the program accepts
|
||||
a net-positive change to replace it with something weaker on remote hosts. That
|
||||
is a user decision (call it D5), not one to assume.
|
||||
|
||||
### G6 clause: test fixtures under production compilation (audited 2026-08-08)
|
||||
|
||||
Audited by importer rather than filename, then checked against what the build
|
||||
actually emits. **The clause is already satisfied on the meaning that matters,
|
||||
and cannot be closed by moving files on the other meaning.**
|
||||
|
||||
36 candidates; 4 have genuine production importers and are correctly placed. The
|
||||
other 32 (~3,300 LOC) have zero non-test importers.
|
||||
|
||||
**They do not ship.** Spot-checked the emitted bundle for
|
||||
`terminal-restore-parity`, `ipcEventsTestHarness`, `storeTestHelpers` and
|
||||
`sshRelayNativeDepsInstallFixture`: none appears in `out/`. Rollup drops them
|
||||
because no production entrypoint reaches them. Under "compiles into the shipped
|
||||
product", this clause holds today.
|
||||
|
||||
**Moving them would close nothing under the other reading.**
|
||||
`config/tsconfig.node.json` and `config/tsconfig.tc.web.json` declare `include`
|
||||
globs such as `../src/main/**/*` with **no `exclude` at all**. A `__tests__/`
|
||||
directory is matched by that glob exactly as any other path is, and so is every
|
||||
`*.test.ts` file in the repo. Relocating 32 fixtures would not remove one file
|
||||
from typecheck scope.
|
||||
|
||||
A sweep was started and stopped once this was verified, rather than landing 32
|
||||
moves across areas this program does not own for no measurable gain.
|
||||
|
||||
If the intent behind the clause is that typecheck scope should exclude test code,
|
||||
that is a repo-wide tsconfig change affecting every test file — different work
|
||||
with a different owner, and it should be stated as such rather than pursued by
|
||||
relocating fixtures.
|
||||
|
||||
One genuine defect was found and fixed while auditing:
|
||||
`terminal-pane/xterm-bypass-event-fixture.ts` and
|
||||
`terminal-pane/__fixtures__/xterm-bypass-event.ts` were byte-identical apart from
|
||||
an import path, and the `__fixtures__` copy had zero importers — a half-finished
|
||||
move left in place. The dead copy is deleted and the live one completed its move.
|
||||
|
||||
## G7 — No regression and reviewable comprehensive change
|
||||
|
||||
**Current status: not started.**
|
||||
|
||||
G7 is proven only on the rebased, converged candidate after G0–G6 and all
|
||||
thirteen journeys are proven. It requires:
|
||||
|
||||
- correctness and security gates;
|
||||
- A/B input latency and output throughput;
|
||||
- backpressure and bounded-memory results;
|
||||
- renderer/app/daemon/relay restore and startup results;
|
||||
- large-pane, long-session, and multi-host scale results;
|
||||
- native packaging/startup on macOS, Linux, and Windows;
|
||||
- WSL, Docker SSH, paired, remote, folder, floating, worktree, drive, and UNC
|
||||
coverage;
|
||||
- both live mixed-version directions;
|
||||
- final categorized LOC census;
|
||||
- independent repository review with no unresolved P0–P2 findings;
|
||||
- release-readiness review with no unresolved correctness, security,
|
||||
compatibility, or performance findings; and
|
||||
- a detailed comprehensive PR whose claims match the receipts.
|
||||
|
||||
Before implementation, record a performance protocol that makes “no
|
||||
regression” falsifiable:
|
||||
|
||||
- fixed candidate and baseline builds, hardware, OS, power mode, network shape,
|
||||
pane/session population, payloads, and background-load policy;
|
||||
- warm-up policy, randomized A/B order, sample count, raw-data location, and
|
||||
statistical method;
|
||||
- input latency, output throughput, memory, backpressure, restore, startup, and
|
||||
large-pane metrics with directionality;
|
||||
- deterministic ceilings for writes, scans, queues, listeners, timers, and
|
||||
allocations on hot paths; and
|
||||
- a predeclared equivalence/no-regression bound no larger than measured baseline
|
||||
noise. A nonzero bound handles measurement noise; it is not permission for a
|
||||
known slowdown and requires explicit user approval.
|
||||
|
||||
Unless an independently reviewed protocol justifies another count, use at least
|
||||
five warm-up trials and thirty measured trials per latency/startup/restore
|
||||
configuration, retain raw samples, and report confidence intervals. Throughput,
|
||||
backpressure, and memory tests must also include a fixed-duration steady-state
|
||||
run and a leak-slope result. Choose all workloads and thresholds before looking
|
||||
at candidate results.
|
||||
|
||||
Green CI, thousands of tests, mergeability, or “reviewable for what shipped” do
|
||||
not prove G7.
|
||||
|
||||
## Thirteen required production journeys
|
||||
|
||||
Every row is currently **not proven**.
|
||||
|
||||
The issue-to-journey matrix in
|
||||
[`related-open-work.md`](./related-open-work.md#mandatory-issue-to-journey-matrix)
|
||||
is part of these journeys, not optional context. Every bound incident needs a
|
||||
red-on-baseline/green-on-candidate oracle, or explicit evidence plus user
|
||||
acceptance that it is unrelated.
|
||||
|
||||
| # | Journey | Required oracle |
|
||||
| --: | ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| 1 | Local macOS, Linux, and Windows | The same pane, full binding, and OS process survive renderer and app restart; every stale exact operation is rejected. Run natively on all three OSes. |
|
||||
| 2 | Daemon and physical WSL | The same PTY survives client and daemon reconnect/restart boundaries; generation skew fails closed without killing or replacing a live successor. |
|
||||
| 3 | Lazy discovery and skipped-host restart | An unused host is not probed eagerly. After a restart that skipped it, lazy rediscovery restores only that host's sessions and never adopts host-current or sibling-host state. |
|
||||
| 4 | Concurrent multi-host connections | At least two distinct final-host connections operate simultaneously. One host's disconnect, CAS, timeout, or failure cannot affect the other. |
|
||||
| 5 | Namespace-partial admission failure | One namespace on a multiplexed connection fails challenge/CAS/grant publication while another commits. Only the failed host-plus-namespace is fenced. |
|
||||
| 6 | Docker OpenSSH with `MaxSessions=1` | The same remote PID and exact binding survive disconnect and client restart. Authority restart imports exactly or exposes unresolved recovery without creating, killing, or adopting. Explicit close or exact proven teardown durably retires the lease and process; unknown ownership stays visible and recoverable rather than being killed. Count actual remote processes, panes, tabs, bindings, leases, and session-cap slots before and after a settle interval. |
|
||||
| 7 | Two independent Docker SSH hosts | Two simultaneous final hosts keep endpoint credentials, principals, namespaces, sessions, cursors, failures, and cleanup completely isolated. |
|
||||
| 8 | Paired client and remote server | The final host remains authoritative across independently updated peers, pairing reconnect, client restart, and remote-runtime restart. |
|
||||
| 9 | Worktree, folder, floating, drive, and UNC namespaces | Each resolves the same stable host-local namespace across spelling/restart changes, without using client repository ID or target ID as identity. |
|
||||
| 10 | Stable proof and exact retry | One bounded device proof identity admits fresh process/session nonces. A lost response retries the exact challenge/request; changed, replayed, cross-host, cross-namespace, or host-current state is rejected. |
|
||||
| 11 | Identity reset and re-enrollment | Crash-resumable host retirement, relay revoke acknowledgement, transport closure, local credential removal, and atomic successor publication occur in order. Offline and old peers remain explicitly pending. |
|
||||
| 12 | Mixed versions in both directions | No unknown opcode or ungranted publication mutates state. Unsupported challenge, grant, delivery, or operation semantics stay on isolated legacy behavior or fail before mutation. |
|
||||
| 13 | Performance and scale | Under the predeclared protocol above, input/output latency, throughput, backpressure, memory, reconnect/restore, startup, large-pane, long-session, and multi-host ceilings show no regression against the fixed baseline. Raw samples, confidence intervals, deterministic counters, and leak slopes satisfy their predeclared bounds. |
|
||||
|
||||
## Cross-cutting correctness cases
|
||||
|
||||
Every relevant journey must exercise:
|
||||
|
||||
- stale, missing, unknown, rejected, timed-out, and disconnected evidence;
|
||||
- concurrent replacement and sibling-host/namespace isolation;
|
||||
- lost request and lost response;
|
||||
- cancellation and partial setup cleanup;
|
||||
- crash immediately before and after every durable boundary;
|
||||
- restart from disk rather than process-memory state;
|
||||
- duplicate and out-of-order delivery;
|
||||
- gaps, overflow, and slow/stalled consumers;
|
||||
- explicit close versus detach;
|
||||
- eventual exact retirement after explicit close or proven teardown while
|
||||
uncertain ownership remains intact and visible;
|
||||
- legacy data with missing optional fields;
|
||||
- relay/daemon incarnation reuse;
|
||||
- sleep/resume and clock movement where timers schedule retries;
|
||||
- exact remote process/PID census, not only persisted-row state; and
|
||||
- cleanup that requires positive identity proof.
|
||||
|
||||
## Forbidden proof substitutions
|
||||
|
||||
- A mock is not a native-platform journey.
|
||||
- A source grep is not a production call-path journey.
|
||||
- A row marked `expired` is not proof that its remote process exited.
|
||||
- Optional pane metadata plus an O(n) repair scan is not structural uniqueness.
|
||||
- An empty list from an unavailable provider is not proof of absence.
|
||||
- A timer or retry budget is not proof of death.
|
||||
- A broad test count is not a correctness oracle.
|
||||
- A test that passes unfixed code is not evidence that a change fixed the bug.
|
||||
- An open or green PR is not shipped behavior.
|
||||
- Historical construction receipts are not proof for a rebased candidate.
|
||||
- Prior art or an uncited LOC comparison cannot delete a requirement.
|
||||
|
||||
## Promotion template
|
||||
|
||||
Before changing any row to `proven`, add a receipt containing:
|
||||
|
||||
```text
|
||||
Gate or journey:
|
||||
Candidate SHA:
|
||||
Merge-base SHA:
|
||||
Unfixed/red SHA:
|
||||
Production path exercised:
|
||||
Exact command or manual protocol:
|
||||
Environment and versions:
|
||||
Expected oracle:
|
||||
Red result:
|
||||
Green result:
|
||||
UTC start/end:
|
||||
Artifact/log:
|
||||
Independent reviewer:
|
||||
Unresolved P0/P1/P2:
|
||||
LOC/performance impact:
|
||||
```
|
||||
|
||||
If any field is missing, the row remains partial or not started.
|
||||
-386
@@ -1,386 +0,0 @@
|
||||
# New design — goalposts
|
||||
|
||||
Tracks the design approved on 2026-08-09 (`design-explained.html`, detail in
|
||||
`counsel-design.html`). Updated as work lands. Progress in every status update
|
||||
is reported against this file.
|
||||
|
||||
## How to read a status
|
||||
|
||||
| Status | Means |
|
||||
| --- | --- |
|
||||
| **PROVEN** | An oracle asserts the behaviour, removing the production guard **reddens** that oracle, and the mutation was verified to actually land before the result was believed. |
|
||||
| **ORACLE RED** | The oracle is written and currently fails for the right reason. The production change is not written yet. This is the intended pre-implementation state. |
|
||||
| **NOT STARTED** | No oracle. |
|
||||
| **BLOCKED** | Waiting on another goalpost or an owner decision. |
|
||||
|
||||
A green test is **not** a status. This program shipped three guards that passed
|
||||
their tests while sitting off the route production takes; "PROVEN" exists to
|
||||
make that impossible to claim by accident.
|
||||
|
||||
---
|
||||
|
||||
## Scoreboard
|
||||
|
||||
| | Count |
|
||||
| --- | --- |
|
||||
| Step goalposts proven | **7 of 7** (one deleted as unreachable) |
|
||||
| Global goalposts proven | **5 of 6** — G1 missed, see below |
|
||||
| Scope removed by evidence | S7 (death rule) — never built |
|
||||
| Oracle clauses green | 26 |
|
||||
| Oracle clauses red (awaiting implementation) | 0 |
|
||||
| Net production lines, correctness work | **+13** |
|
||||
| Net production lines, incl. the approved affordance | **+83** |
|
||||
| Net test lines | +2,578 |
|
||||
|
||||
**G1 is not met, and that is reported rather than smoothed over.** The design's
|
||||
net-negative target assumed the remaining steps were collapses. Three were
|
||||
(S3 −21, S8 −32, and the relay half of S5), but two genuinely add code: the
|
||||
single `bindPaneShell` producer (+60, since one function replaces three
|
||||
divergent call sites without those sites shrinking much) and the D1
|
||||
disconnected-pane affordance (+70), which is a new product surface the owner
|
||||
approved — a feature, not a refactor. Excluding the affordance the correctness
|
||||
work is +13, i.e. roughly a wash. See "G1" below for the full accounting.
|
||||
|
||||
---
|
||||
|
||||
## Step goalposts
|
||||
|
||||
### S1 — An identity mismatch is never read as death · **PROVEN**
|
||||
|
||||
*Guarantee.* The relay reports a pane-identity mismatch by saying the pty was
|
||||
not found. It found it — comparing is how it noticed. That must never reach a
|
||||
respawn decision as proof of death.
|
||||
|
||||
- Oracle: `src/main/providers/ssh-pty-identity-mismatch-is-not-death.test.ts`,
|
||||
`src/renderer/src/components/terminal-pane/reattach-failure-classification.test.ts`
|
||||
- Clauses: 7 + 3
|
||||
- Mutations proven: restoring the destructive wrap reddens 2 clauses; deleting
|
||||
the classifier guard reddens 1 — clause-selective, each proven separately.
|
||||
- Landed: `e2524b0472f`
|
||||
|
||||
### S2 — Pane identity is not sent on reattach · **PROVEN**
|
||||
|
||||
*Guarantee.* Moving a pane to another tab must never make its terminal
|
||||
unreachable.
|
||||
|
||||
- Oracle: same file, `reattach does not ask the relay to police pane identity` (2 clauses)
|
||||
- Also inverted an existing test that pinned the removed behaviour, so the new
|
||||
intent stays covered rather than silently dropped.
|
||||
- Works against **already-deployed relays**: the relay's comparison is
|
||||
presence-guarded, so not sending the fields disarms it everywhere. No wire
|
||||
change, no redeploy.
|
||||
- Landed: `c51be8072ba`
|
||||
|
||||
### S3 — A failed reattach never fabricates an exit · **PROVEN**
|
||||
|
||||
*Guarantee.* On an unproven not-found, the pane is not told the program exited,
|
||||
ownership is not deleted, provider state is not cleared, and the lease is not
|
||||
expired. The case routes into the non-destructive recovery branch that already
|
||||
exists.
|
||||
|
||||
- Oracle: `src/main/ssh/ssh-relay-reattach-exit-proof.test.ts` — 8 green
|
||||
- Mutation proven: restoring the destructive block reddens 6 of 8 clauses; the
|
||||
2 producer pins stay green, so the mutation is clause-selective.
|
||||
- Two tests pinned the deleted premise ("attach verifies liveness before
|
||||
answering not-found") and were **inverted**, not patched.
|
||||
- Shipped with the disconnected-pane affordance below, since panes now stay
|
||||
visible instead of silently respawning.
|
||||
- Landed: `c19f1b386b4` (−21 production lines)
|
||||
|
||||
### S4 — One partition per (target, pane) · **PROVEN**
|
||||
|
||||
*Guarantee.* A pane that re-leases under a new relay id across reconnects ends
|
||||
with exactly **one** live claim; the predecessor is superseded, not left live.
|
||||
|
||||
- Oracle: `src/main/ssh-pane-binding-partition.test.ts` — 7 green (one clause added)
|
||||
- The highest-value goalpost: the mechanism behind the reported 2 → 19 → 20.
|
||||
- Mutation proven as a **2×2**, because the two edits mask each other:
|
||||
|
||||
| reader | load fold | result |
|
||||
| --- | --- | --- |
|
||||
| hedge (original) | off | 6 red — baseline |
|
||||
| hedge | off | 4 red — the 2 source-text clauses now green |
|
||||
| local-only | off | 1 red — the reader carries 3 clauses |
|
||||
| hedge | **on** | **all green** ← the fold masks the reader |
|
||||
| local-only | on | all green |
|
||||
|
||||
That fourth row is why an **eighth clause was added**: with the fold shipping,
|
||||
the fold erases the divergent copy at boot, so the reader guard would have
|
||||
shipped unproven — precisely the failure mode G5 exists to catch. The new
|
||||
clause rewrites `ssh:<target>` mid-session (orphan adoption still writes
|
||||
there) and reddens when the hedge is restored, pinning the reader on its own.
|
||||
- Five clauses in `ipc/pty.test.ts` pinned the two-partition shape and were
|
||||
**inverted** to the single home, each keeping an arity check so a re-added
|
||||
partition argument fails loudly.
|
||||
- Side effect found while verifying: the renderer never hydrated the `ssh:*`
|
||||
partition (`listKnownRuntimeHostIds` filters to `runtime:*`), so the Issue
|
||||
#217 force-quit binding protection had never worked for SSH panes. It does now.
|
||||
- Landed: `2733c59879b` (+9 production lines — the call sites shrank; the fold
|
||||
is new state repair)
|
||||
|
||||
### S5 — The superseded-pane fence is live on the reattach path · **PROVEN**
|
||||
|
||||
*Guarantee.* After a relay-driven reattach binds a pane, a stale write aimed at
|
||||
the superseded predecessor is **refused**. It used to be permitted, because the
|
||||
fence's bookkeeping was only written by spawn.
|
||||
|
||||
- Oracle: `src/main/ssh/ssh-relay-session-reattach-pane-fence.test.ts` — 4 green
|
||||
(one clause added)
|
||||
- Collapsed to one `bindPaneShell` producer used by the relay reattach and both
|
||||
spawn handlers. Error policy stays at the call sites because it genuinely
|
||||
differs: a caller that just created a shell must clean it up on a failed
|
||||
durable write; a caller that merely reattached must not detach anything.
|
||||
- Mutation proven, both sub-guards isolated:
|
||||
- drop the `rememberPaneKeyForPty` call → 3 clauses redden
|
||||
- prefer the stored `tabId` over the live layout → 1 clause reddens
|
||||
- That second clause is **new**. Every pre-existing clause used one tabId on
|
||||
both sides, so a producer that simply forwarded `lease.tabId` would have gone
|
||||
green and shipped the moved-pane bug unnoticed. Only the leaf half of a pane
|
||||
key is remint-stable; `detachTerminalPaneToTab` moves a live pane and its PTY.
|
||||
- Two source-text clauses were **strengthened**: they used to require the relay
|
||||
to hold a `persistPtyBinding` call and merely forbade an ssh-partition
|
||||
argument. The relay now has none, so they assert **zero** direct binding
|
||||
writes there — a second bind producer is exactly the defect being removed.
|
||||
- Repaired a latent false green: the "persistence fails" case in
|
||||
`ssh-relay-session-reconnect-incarnation` was passing because a missing mock
|
||||
made the call throw a TypeError that happened to emit the asserted
|
||||
`console.error`. The failure is now injected at the producer.
|
||||
- Landed: `0f409055d3f` (+60 production lines — the one step that grows)
|
||||
|
||||
### S6 — Settle whether the 30s recovery grant executes at all · **PROVEN — it is dead code**
|
||||
|
||||
*Result.* It cannot execute for a real SSH pane. Verified personally:
|
||||
|
||||
- The lease stores a **relay-native** pty id, normalized on write
|
||||
(`persistence.ts:7184`, comment: "app ids are global").
|
||||
- The caller passes the **app-form** id (`orca-runtime.ts:16478` ← `toAppSshPtyId`).
|
||||
- The comparison is raw `lease.ptyId === ptyId` (`orca-runtime.ts:6310`) with no
|
||||
normalization, so the two forms can never be equal.
|
||||
- The branch is also unreachable for a local pane, since it requires an SSH lease.
|
||||
- Its covering test seeds both sides as the same literal with a null
|
||||
connectionId — a shape production cannot produce.
|
||||
|
||||
### S7 — The death rule · **DELETED on S6's evidence**
|
||||
|
||||
Not built. The arbitration machinery existed to referee a branch that never
|
||||
executes. Per the design: "if E-1 shows the grant branch unreachable, E-2 is
|
||||
deleted and the marker simply never fires, which is the safe end state."
|
||||
|
||||
### S8 — Remove the dead recovery-grant path · **PROVEN**
|
||||
|
||||
*Guarantee.* Deleting it changes no behaviour, because it has none.
|
||||
|
||||
- Oracle: `src/main/runtime/ssh-pane-recovery-grant-reachability.test.ts`, written
|
||||
and green **before** the deletion, so inertness is demonstrated rather than assumed.
|
||||
- It mints **both** id forms from the production helpers rather than as two
|
||||
hand-typed literals, so it tracks the real namespace split instead of
|
||||
restating it, and seeds a lease qualifying on every other predicate (state,
|
||||
worktree, tab, leaf, grace window). Anti-vacuity clauses pin that control
|
||||
actually reaches the gate rather than bailing out earlier.
|
||||
- Mutation proven **before** deleting: normalizing the `lease.ptyId === ptyId`
|
||||
comparison makes the grant fire and reddens the oracle. That is the exact fix
|
||||
someone would reach for, so the oracle is pinned to unreachability, not to the
|
||||
throw.
|
||||
- Deleted: the grant tail, the `terminalPaneRecoveryByIdentity` dedup map (whose
|
||||
only consumer was the grant), and the dead `ptyId` parameter.
|
||||
- **Not** deleted, because over-deleting here breaks users: `getRecentExpiredSshLease`,
|
||||
`hasRecentExpiredSshLeasePane` and `SSH_PANE_RECOVERY_GRACE_MS` all stay. Their
|
||||
other two callers pass `ptyId` undefined, which short-circuits the broken
|
||||
comparison — those are live and feed headless-mobile terminal-tab visibility.
|
||||
- Also **not** done: deleting only the gate while keeping the spawn. That would
|
||||
have granted a respawn to every disconnected pane — a change in the dangerous
|
||||
direction. The refusal is what stays.
|
||||
- Three tests pinned the grant and were **inverted**; a fourth is now
|
||||
tautological and carries a comment saying so rather than being left hollow.
|
||||
- Honest limit, recorded in the oracle: unreachable **by construction** for SSH
|
||||
panes; for a local pane, unreachable only up to a random-UUID collision.
|
||||
- Landed: `7d44315437a` (−32 production lines)
|
||||
|
||||
### Product affordance (D1) — **SHIPPED with S3**
|
||||
|
||||
A pane that cannot be verified renders as disconnected with two explicit actions
|
||||
— "Try again" and "Start a new terminal" — instead of silently respawning.
|
||||
|
||||
- No new IPC channel: the silent-respawn decision was always renderer-local.
|
||||
Both actions are things the code already did, moved behind a click.
|
||||
- Copy constraint enforced **as an oracle**, not as a review note: the rendered
|
||||
text must match no death verb and show no wire token. STYLEGUIDE.md:236 already
|
||||
forbids result verbs without result data, and a failed attach is not result data.
|
||||
- `TerminalRemoteRuntimeReconnectBanner` → `TerminalPaneDisconnectedBanner`; it
|
||||
now serves any transport. Existing i18n key strings kept verbatim so no shipped
|
||||
translation breaks; the SSH copy is additive.
|
||||
- Landed: `5e1b57bb2f9` (+70 production lines)
|
||||
|
||||
---
|
||||
|
||||
## Global goalposts
|
||||
|
||||
### G1 — Net production code is negative · **NOT MET (+83 this phase)**
|
||||
|
||||
Counted from the pre-work merge base, production only. Reported as a miss rather
|
||||
than reframed until it passes.
|
||||
|
||||
| Step | Net production |
|
||||
| --- | --- |
|
||||
| S3 — collapse the fabricated exit | **−21** |
|
||||
| S4 — one partition | +9 |
|
||||
| S5 — one bind producer | **+60** |
|
||||
| S8 — delete the dead grant | **−32** |
|
||||
| D1 affordance (new product surface) | **+70** |
|
||||
| Banner rename (pure move, no behaviour) | +36 of the above, not real growth |
|
||||
| **Correctness work only (S3–S8)** | **+13** |
|
||||
| **Total this phase** | **+83** |
|
||||
|
||||
Why it missed. The target assumed the remaining steps were collapses. Three were.
|
||||
Two were not, for reasons worth stating:
|
||||
|
||||
- **S5 (+60).** One producer replacing three divergent call sites only shrinks
|
||||
the code if those sites were duplicating the whole bind. They were not — they
|
||||
each did *part* of it differently, which is exactly the defect. The producer is
|
||||
net-new; the call sites shrank by 7 lines. Collapsing the two spawn handlers'
|
||||
surrounding logic as well would buy the difference back, but it would reorder
|
||||
side effects that existing ordering assertions pin, so it was left alone.
|
||||
- **D1 affordance (+70).** A new user-facing surface the owner approved. Features
|
||||
add lines; counting it against a refactor budget would be the wrong pressure —
|
||||
the way to make this number go negative would be to delete the affordance.
|
||||
|
||||
Where the remaining slack is, if the goal is to be met later: steps P and K of
|
||||
the design (the leaf-keyed record replacing `SshRemotePtyLease`, and the lease
|
||||
readers ported onto it) are the deletions this phase deferred.
|
||||
|
||||
### G2 — No redeploy required for the user-visible fixes · **PROVEN for S1–S8**
|
||||
|
||||
Fixes must work against relays already installed on people's hosts. Every step
|
||||
that landed is client-side only:
|
||||
|
||||
- S1, S2 — proven previously.
|
||||
- S3 — deletes client-side reactions to a relay message; the relay is unchanged.
|
||||
- S4 — client-local persistence layout only; nothing crosses the wire.
|
||||
- S5 — client-local in-memory fence maps and the durable binding.
|
||||
- S8 — deletes a client-side branch.
|
||||
- D1 — renderer only.
|
||||
|
||||
**No relay redeploy is required for any of it**, which matters because the
|
||||
reported failure is happening on hosts people have already deployed.
|
||||
|
||||
### G3 — Wire compatibility · **SATISFIED — nothing on the wire changed**
|
||||
|
||||
Clients and hosts update independently. This phase added **no** RPC parameter,
|
||||
no stream opcode and no published field, so there is nothing to negotiate. The
|
||||
orphan projection that would have added one optional field belongs to step W,
|
||||
which is not in this phase.
|
||||
|
||||
One wire-adjacent behaviour change is worth naming for reviewers: `terminal.recoverPane`
|
||||
(RPC) now always refuses for a disconnected pane. The method still exists and its
|
||||
signature is unchanged, so older clients get a refusal rather than a
|
||||
`method_not_found` — and the renderer already handles refusal, because that is
|
||||
what the branch did in practice anyway.
|
||||
|
||||
### G4 — Cross-platform · **SATISFIED by construction**
|
||||
|
||||
macOS, Linux, Windows, WSL. No oracle added in this phase shells out, and none
|
||||
uses `echo $$` or `ps` — they assert against persisted state, in-memory maps and
|
||||
rendered text. No rule that landed concludes anything from a relay restart or
|
||||
from whether a shell survives a hard relay death, which is the Windows-divergent
|
||||
question the design deliberately refuses to depend on. The affordance copy holds
|
||||
on every platform precisely because it never claims the shell died.
|
||||
|
||||
### G5 — Every guard is proven live on the production route · **PROVEN for S1–S8**
|
||||
|
||||
For each new guard, an oracle must redden when the **producer** is removed — not
|
||||
only when the guard itself is removed.
|
||||
|
||||
This exists because it had failed three times here: an inert `mayCreate`, a
|
||||
keystroke fence inert on reattach, and a respawn gate on a minority path. S5 is
|
||||
the remediation of the second.
|
||||
|
||||
**It nearly failed a fourth time in this phase, and the rule caught it.** S4's
|
||||
reader guard (`durablyBoundPtyIdForPane` reading one partition) initially showed
|
||||
*no* mutation response: restoring the ssh-first hedge left all six clauses green,
|
||||
because the load fold had already erased the divergent copy at boot. The guard
|
||||
was correct and would have shipped unproven. The fix was an added clause that
|
||||
rewrites the ssh partition **mid-session**, which reddens on the hedge and pins
|
||||
the reader independently of the fold.
|
||||
|
||||
Two further near-misses found and closed the same way:
|
||||
- S5's clauses all used one tabId on both sides, so "compose the paneKey from
|
||||
the current tab" was unpinned — a producer forwarding `lease.tabId` would have
|
||||
gone green. A moved-pane clause now pins it.
|
||||
- A case in `ssh-relay-session-reconnect-incarnation` was a **latent false
|
||||
green**: it passed because a missing mock threw a TypeError that happened to
|
||||
emit the `console.error` it asserted. Now injected at the producer.
|
||||
|
||||
### G6 — No `max-lines` disables · **HOLDING**
|
||||
|
||||
Per AGENTS.md. No per-file bumps either.
|
||||
|
||||
---
|
||||
|
||||
## Definition of done
|
||||
|
||||
| | Criterion | Status |
|
||||
| --- | --- | --- |
|
||||
| 1 | S1–S5 and S7 PROVEN, or S7 deleted on S6's evidence | **MET** — S1–S6 and S8 proven; S7 deleted on S6's evidence |
|
||||
| 2 | G1–G6 satisfied, each with evidence recorded here | **PARTIAL** — G2–G6 met; **G1 missed at +83**, accounted for above |
|
||||
| 3 | The reported failures covered by an oracle that reddens without the fix | **MET.** Pane cardinality: S4's ten-reconnect clause, plus an E2E spec driving three real reconnects against a Docker relay (passes). Duplicate resume: the classification clauses, plus an E2E spec where the **host itself** records every shell launch (passes). Writing the second one is what exposed that RC2 was still live — see below |
|
||||
| 4 | No guard shipped without a producer-side mutation proving it is reachable | **MET** — and it caught one guard that would otherwise have shipped unproven (see G5) |
|
||||
|
||||
### Found after the goalposts were met
|
||||
|
||||
Two defects surfaced *after* every step goalpost was proven, and both were invisible to the unit
|
||||
oracles that had already gone green. Recorded because they are the most useful evidence in this
|
||||
phase about where these oracles stop reaching.
|
||||
|
||||
**RC2 survived S3 — found by writing the E2E test.** `7cd7fef927f`.
|
||||
The reported duplicate agent resume was not fixed. `reattachSshPtySession` maps every relay
|
||||
not-found to `SSH_SESSION_EXPIRED`, and `isProvenSshSessionGoneError` returned true for it, so the
|
||||
renderer's reattach arm still respawned. The E2E harness produced the shape directly: a stalled
|
||||
relay is superseded by a fresh one with no memory of `pty-1`, while the old shells keep running
|
||||
under the stopped process. The new relay answers not-found for a live shell.
|
||||
|
||||
A not-found proves an exit only if the relay process that minted the pty is the one answering —
|
||||
design D3 row 2, gated on `relayInstanceId`, which step E-2 never built. `SSH_SESSION_EXPIRED` is
|
||||
not independent evidence: its only producer is that same mapping, and the token's own doc comment
|
||||
claimed "the host proved the session is gone", which it never did.
|
||||
|
||||
So `isProvenSshSessionGoneError` returns false. This is also what makes the D1 affordance
|
||||
load-bearing rather than near-unreachable. The respawn tails are deliberately kept — the design
|
||||
preserves the grant as a conditional for E-2 — with a clause pinning that nothing reaches them.
|
||||
|
||||
**The fold left half a fence behind — found by adversarial review.** `994733d8b1a`.
|
||||
`persistPtyBinding` writes the binding and its incarnation into the same partition, and the
|
||||
incarnation is what its CAS compares. The fold moved only `ptyIdsByLeafId`, so after upgrade the
|
||||
guard stayed in the partition nothing reads and the CAS compared `undefined` against `undefined`.
|
||||
Not data loss and not a wrong-shell bind, but a guard silently weakened by a migration is the exact
|
||||
shape this program keeps finding.
|
||||
|
||||
### Residuals, stated rather than closed quietly
|
||||
|
||||
- **Orphan adoption still writes the `ssh:<target>` partition.**
|
||||
`adoptTerminalOrphansFromInventory` (via `tryGetWorkspaceSessionHostIdForWorktree`)
|
||||
writes pane bindings there. S4's reader ignores that partition, so supersession
|
||||
is unaffected and the guarantee holds — but the write now lands somewhere
|
||||
nothing reads. Harmless today, and a trap for whoever touches it next. Belongs
|
||||
with step W.
|
||||
- **The load fold moves leaf bindings only.** It deliberately does not touch
|
||||
`tabsByWorktree[*].ptyId` in the ssh partition: that field is a tab-level
|
||||
pointer no supersession path reads, and nulling it without a local counterpart
|
||||
to move it to would be data loss. Narrowing the fold to what the defect
|
||||
actually requires is why `reassignSshTargetId` needed no inversion.
|
||||
- **The main-side detached branch still has no renderer signal.** S3 made the
|
||||
main process stop lying; the affordance covers the renderer-initiated reattach
|
||||
arms. A relay-driven reattach failure remains silent to the user. Closing it
|
||||
needs a `pty:detached` channel — deferred deliberately, not overlooked.
|
||||
|
||||
## Deliberately out of scope
|
||||
|
||||
- The data-plane work (transport delivery guarantee, binary payloads, one credit
|
||||
ledger). Real and separately justified, but independent of these goalposts.
|
||||
- Rebuilding the authority architecture rejected earlier at +60,903 lines.
|
||||
|
||||
## Owner decisions
|
||||
|
||||
| | Decision | Status |
|
||||
| --- | --- | --- |
|
||||
| D1 | An unverifiable pane becomes visibly disconnected with two actions, instead of silently respawning | **Approved 2026-08-09** (implicit in approving the design; flagged for correction if not intended) |
|
||||
| D2 | Whether the older gate/journey framing is rescoped or retired now that this design supersedes it | **Open** |
|
||||
@@ -1,341 +0,0 @@
|
||||
# Related GitHub work
|
||||
|
||||
This is a curated overlap map, not a merge queue. States were verified against
|
||||
GitHub on 2026-08-08 UTC and must be refreshed before any rebase, merge, close,
|
||||
or implementation decision.
|
||||
|
||||
Rules:
|
||||
|
||||
- Do not stack PRs merely because they appear in the same section.
|
||||
- “Conflict” means semantic or file overlap that requires one coherent design,
|
||||
not an instruction to merge both.
|
||||
- A related PR can supply diagnosis, tests, or a small reusable primitive
|
||||
without becoming a dependency.
|
||||
- Landing a related PR does not promote G0–G7 or one of the thirteen journeys
|
||||
unless the final converged tree passes that gate's complete proof.
|
||||
- GitHub issue/PR text is context, not authority over the user's goal.
|
||||
|
||||
## Immediate incident and competing implementations
|
||||
|
||||
- [Issue #11729 — SSH remote environment not restored after login on Rocky Linux
|
||||
9.4](https://github.com/stablyai/orca/issues/11729) — **open**. This is the
|
||||
named STA-3077 customer incident. Its final acceptance must include Rocky
|
||||
Linux SSH, repeated reconnect, the same PID/process/session/transcript
|
||||
identity, restored workspace environment, visible pane count, and actual
|
||||
remote relay PTY/process count.
|
||||
- [PR #13111 — stop reconnect grafting panes, stacking leases, and respawning
|
||||
live shells](https://github.com/stablyai/orca/pull/13111) — **open**, current
|
||||
branch, green checks, no approval. It is narrow containment and leaves remote
|
||||
shell reclamation unresolved. One major Docker settle review thread remains
|
||||
[unresolved](https://github.com/stablyai/orca/pull/13111#discussion_r3739665033).
|
||||
- [PR #12264 — stop reconnect from grafting ghost terminal
|
||||
panes](https://github.com/stablyai/orca/pull/12264) — **open and conflicting**.
|
||||
It targets the same incident and overlaps persistence, PTY, SSH relay, and E2E
|
||||
files. Mine its diagnosis and discriminating artifacts; do not stack it
|
||||
wholesale with #13111.
|
||||
- [PR #12743 — contain reattach to existing panes, quarantine
|
||||
orphans](https://github.com/stablyai/orca/pull/12743) — **open and
|
||||
conflicting**. It shares `mayCreate: false` and broader relay coverage, but
|
||||
quarantine must be judged against the no-reconciliation final design.
|
||||
- [PR #12456 — reattach on `restoreRequired` instead of expiring a live
|
||||
session](https://github.com/stablyai/orca/pull/12456) — **open**. It addresses
|
||||
the #12448 branch through retry rather than cold respawn. Treat it as a
|
||||
semantic alternative or complementary recovery path only after choosing one
|
||||
exact restore protocol.
|
||||
- [PR #9820 — reap orphaned relay PTYs before session-cap
|
||||
exhaustion](https://github.com/stablyai/orca/pull/9820) — **open**. Its
|
||||
time-based remote kill conflicts with unknown-is-not-dead and #13111's
|
||||
deliberate non-kill behavior. Use it as incident evidence, not a safe default
|
||||
dependency.
|
||||
- [PR #12798 — opt-in zmx-backed durable SSH
|
||||
terminals](https://github.com/stablyai/orca/pull/12798) — **open**. This is a
|
||||
broad architectural alternative for final remote PTY ownership with direct
|
||||
file overlap. Evaluate capability-by-capability; never blend it into the
|
||||
current model accidentally.
|
||||
|
||||
## Open incident issues that should become explicit oracles
|
||||
|
||||
- [Issue #9819 — SSH relay leaks orphaned PTYs until the 50-session
|
||||
cap](https://github.com/stablyai/orca/issues/9819) — explicit close,
|
||||
disconnected close with a lost shutdown response, and worktree teardown must
|
||||
durably retry retirement of the exact lease and process. Uncertain sessions
|
||||
must survive visibly until resolved. Repeated reconnect must reclaim actual
|
||||
relay PIDs/session slots rather than accumulate toward the cap.
|
||||
- [Issue #9034 — SSH reconnect repeatedly spawns detached PTYs for one
|
||||
pane](https://github.com/stablyai/orca/issues/9034) — count actual remote PTYs
|
||||
and prove bounded one-pane ownership across repeated reconnect.
|
||||
- [Issue #11006 — transient SSH disconnect treats live relay PTYs as dead and
|
||||
duplicate-respawns agents](https://github.com/stablyai/orca/issues/11006) —
|
||||
disconnect with a live process must remain unresolved/reattachable and must
|
||||
not start another agent.
|
||||
- [Issue #12447 — closed SSH tabs resurrect and auto-resume old
|
||||
sessions](https://github.com/stablyai/orca/issues/12447) — explicit close must
|
||||
persist its pending tombstone before a kill RPC, survive a lost reply plus app
|
||||
and relay restart, retry after handshake, reclaim the exact remote PTY, and
|
||||
never recreate UI or automatic resume.
|
||||
- [Issue #12448 — `restoreRequired` is misreported as session
|
||||
expiry](https://github.com/stablyai/orca/issues/12448) — source recovery must
|
||||
replay/repaint on the same PID and binding without starting a replacement
|
||||
`--resume`; identity drift, timeout, or unknown state must not become proof of
|
||||
process death or cold resume authority.
|
||||
- [Issue #12699 — SSH sleeping-agent wake sweep forks live remote
|
||||
sessions](https://github.com/stablyai/orca/issues/12699) — all wake/resume
|
||||
paths must share the same exact liveness and identity contract.
|
||||
- [Issue #12683 — remote-runtime disconnect spends recovery on stale same-handle
|
||||
reattach](https://github.com/stablyai/orca/issues/12683) — paired/web terminal
|
||||
recovery must fence generations and retain automatic liveness.
|
||||
- [Issue #10208 — duplicate terminal tab on worktree session
|
||||
restore](https://github.com/stablyai/orca/issues/10208) — local/worktree
|
||||
restore must prove the same pane and process rather than creating a duplicate.
|
||||
|
||||
These issues are separate customer-visible symptoms. A single structural model
|
||||
should make their shared invalid transitions impossible, while each issue keeps
|
||||
its own discriminating regression journey.
|
||||
|
||||
## Daemon, relay generation, and liveness work
|
||||
|
||||
The acceptance matrix must also reproduce these open lifecycle incidents:
|
||||
|
||||
- [Issue #11904 — tab close with a broken binding orphans the daemon
|
||||
session](https://github.com/stablyai/orca/issues/11904) — close intent must be
|
||||
durable before transport work and eventually retire the exact process.
|
||||
- [Issue #8585 — detached relay generations are never
|
||||
reaped](https://github.com/stablyai/orca/issues/8585) — termination must name
|
||||
the exact relay generation; path/PID reuse or a failed connection cannot kill
|
||||
a successor.
|
||||
- [Issue #9138 — updates leave old daemon generations and sessions
|
||||
alive](https://github.com/stablyai/orca/issues/9138) and
|
||||
[issue #11342 — stale daemon/PTY generations leak across upgrade and
|
||||
close](https://github.com/stablyai/orca/issues/11342) — upgrade must converge
|
||||
without invisible sessions, guessed ownership, or unbounded retention.
|
||||
- [Issue #10415 — Windows daemon crash on unkillable PTY and protocol
|
||||
bump](https://github.com/stablyai/orca/issues/10415) — native Windows must
|
||||
preserve live old-generation sessions or expose exact unresolved recovery;
|
||||
one unkillable PTY cannot crash the daemon.
|
||||
- [Issue #8275 — worktree teardown kills unrelated split-pane
|
||||
sessions](https://github.com/stablyai/orca/issues/8275) — teardown and failure
|
||||
must remain exact-namespace/exact-binding scoped.
|
||||
|
||||
- [PR #12882 — publisher-owned daemon endpoint
|
||||
replacement](https://github.com/stablyai/orca/pull/12882) — **open and
|
||||
mergeable** when checked. It provides single-writer endpoint publication and
|
||||
three-valued liveness. Reconcile it before editing overlapping daemon paths;
|
||||
do not assume the older handoff's merge order is still current.
|
||||
- [PR #11622 — fence daemon session ownership across
|
||||
generations](https://github.com/stablyai/orca/pull/11622) — **open** and held
|
||||
behind further audit per its description. Strong explicit
|
||||
owned/unavailable/ambiguous semantics, but broad overlap makes it design input,
|
||||
not an automatic dependency.
|
||||
- [PR #9833 — report local PTY inventory
|
||||
readiness](https://github.com/stablyai/orca/pull/9833) — **open**. Its
|
||||
pending/ready/stale distinction is reusable context for unknown-is-not-dead.
|
||||
- [PR #12760 — reap only the proven relay
|
||||
generation](https://github.com/stablyai/orca/pull/12760) — **open**. It adds
|
||||
exact relay generation ownership before termination and conflicts in
|
||||
`ssh-relay-session.ts`; reconcile before designing remote shell cleanup.
|
||||
- [PR #8618 — reap a detached relay after failed
|
||||
reconnect](https://github.com/stablyai/orca/pull/8618) — **open** and an older
|
||||
competing predecessor to #12760. Do not combine both ownership schemes.
|
||||
- [PR #12749 — wait for a disconnected PTY owner's full
|
||||
grace](https://github.com/stablyai/orca/pull/12749) — **open**. Useful owner
|
||||
admission and grace context; timers may schedule retry but cannot independently
|
||||
authorize takeover or destruction.
|
||||
- [PR #12702 — stop sleeping-agent wake sweep from forking live SSH
|
||||
sessions](https://github.com/stablyai/orca/pull/12702) — **open**. Complementary
|
||||
#12699 work that must converge on the same ownership/liveness behavior; it
|
||||
need not share an internal primitive unless the reconciled design calls for
|
||||
one.
|
||||
- [PR #10118 — durable persistence write
|
||||
seam](https://github.com/stablyai/orca/pull/10118) — **open**. It overlaps
|
||||
`persistence.ts` and is relevant to durability and main-thread performance;
|
||||
reconcile before large persistence changes.
|
||||
|
||||
## Paired runtime, remote server, and E2EE lifecycle work
|
||||
|
||||
These open issues are mandatory inputs to the paired/remote/platform journeys:
|
||||
|
||||
- [Issue #11495 — paired viewer loses attachments across update/restart while
|
||||
host PTYs remain alive](https://github.com/stablyai/orca/issues/11495) — the
|
||||
reported large host population is a strong restore/scale oracle; each pane
|
||||
must return to the same host PTY/process.
|
||||
- [Issue #11265 — paired terminals stall despite live TCP
|
||||
sockets](https://github.com/stablyai/orca/issues/11265) — transport connectivity
|
||||
is not delivery liveness; recovery must resume output without an unrelated RPC
|
||||
kick.
|
||||
- [Issue #11803 — remote stop respawns tabs from a stale
|
||||
owner](https://github.com/stablyai/orca/issues/11803) — exact durable retirement
|
||||
must remain retired after host/client restart.
|
||||
- [Issue #12241 — paired-host session partitions grow without
|
||||
bound](https://github.com/stablyai/orca/issues/12241) — paired and scale
|
||||
journeys must prove bounded state and safe lost-consumer
|
||||
retirement/compaction liveness.
|
||||
- [Issue #11574 — same-server re-pairing changes host identity and strands
|
||||
sessions](https://github.com/stablyai/orca/issues/11574) — stable final-host
|
||||
identity must survive re-pairing without adopting another host.
|
||||
- [Issue #9827 — investigate WSL session
|
||||
restoration](https://github.com/stablyai/orca/issues/9827) and
|
||||
[issue #11339 — WSL commands are not restored after
|
||||
restart](https://github.com/stablyai/orca/issues/11339) — the daemon/WSL
|
||||
journey must run on physical WSL and preserve both shell commands and agent
|
||||
sessions.
|
||||
|
||||
Additional open incident inputs must be triaged into a named journey or ruled
|
||||
unrelated with explicit evidence and user acceptance:
|
||||
|
||||
| Issue | Required question |
|
||||
| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
|
||||
| [#11800 — paired null-PTY terminal ghosts](https://github.com/stablyai/orca/issues/11800) | Does host topology advancement retire exact placeholders without inference or a permanent timer reconciler? |
|
||||
| [#9092 — remote runtime fails after macOS sleep](https://github.com/stablyai/orca/issues/9092) | Does generation-fenced reconnect recover after sleep with bounded resources and no identity change? |
|
||||
| [#9585 — remote ghost tabs after host restart](https://github.com/stablyai/orca/issues/9585) | Are exit/retirement outcomes durable and idempotent across host restart? |
|
||||
| [#9562 — remote host session restarts empty](https://github.com/stablyai/orca/issues/9562) | Does the complete boundary plus replay restore terminal state after server restart? |
|
||||
| [#12568 — no PTY provider after SSH relay recovery/update](https://github.com/stablyai/orca/issues/12568) | Does lazy rediscovery restore the exact host/provider generation? |
|
||||
| [#10385 — mobile remains falsely connected after RPC stalls](https://github.com/stablyai/orca/issues/10385) | Can E2EE/mobile liveness recover when transport looks connected but RPC is stalled? |
|
||||
| [#12140 — mobile pairing socket closes immediately](https://github.com/stablyai/orca/issues/12140) | Do capability, identity, and E2EE negotiation fail explicitly and recover across LAN/Tailscale? |
|
||||
| [#8129 — mobile events lost while disconnected](https://github.com/stablyai/orca/issues/8129) | Are semantic outcomes durable and replayed to a disconnected/backgrounded consumer? |
|
||||
|
||||
- [PR #11575 — preserve host identity across same-server
|
||||
re-pairing](https://github.com/stablyai/orca/pull/11575) — **open**. Direct
|
||||
identity context for the paired reconnect journey.
|
||||
- [PR #12768 — restore paired snapshots at the source
|
||||
grid](https://github.com/stablyai/orca/pull/12768) — **open**. Complementary
|
||||
host-snapshot authority work with renderer file overlap.
|
||||
- [PR #9093 — recover remote runtime streams after client
|
||||
wake](https://github.com/stablyai/orca/pull/9093) — **open**. Provides
|
||||
generation-cancelled bounded-backoff reconnect context.
|
||||
- [PR #10235 — self-heal remote subscriptions after server
|
||||
restart](https://github.com/stablyai/orca/pull/10235) — **open**. Relevant to
|
||||
E2EE session loss hidden by a live relay transport.
|
||||
- [PR #11596 — recover falsely connected mobile RPC
|
||||
sessions](https://github.com/stablyai/orca/pull/11596) — **open and
|
||||
conflicting**. Broad mobile/E2EE reconnect context; do not absorb its parallel
|
||||
lifecycle machinery without a full state-machine audit.
|
||||
- [PR #12036 — stop blocking E2EE handshake on synchronous ACL
|
||||
writes](https://github.com/stablyai/orca/pull/12036) — **open and conflicting**.
|
||||
Preserve its metadata-versus-correctness durability distinction and measured
|
||||
Windows performance constraint during identity lifecycle work.
|
||||
- [PR #12134 — refresh stale capabilities before
|
||||
pairing](https://github.com/stablyai/orca/pull/12134) — **open**. Capability
|
||||
freshness and reauthentication are required mixed-version/pairing context.
|
||||
- [PR #11804 — make remote stop retirement
|
||||
durable](https://github.com/stablyai/orca/pull/11804) — **open**. Strong
|
||||
complementary evidence for exact persisted owner routing and host-side
|
||||
retirement acknowledgement.
|
||||
- [PR #12987 — distinguish an absent worktree from an empty tab
|
||||
list](https://github.com/stablyai/orca/pull/12987) — **open**. Reusable
|
||||
unknown-versus-authoritative-empty semantics.
|
||||
- [PR #12751 — preserve tabs when a snapshot reports
|
||||
none](https://github.com/stablyai/orca/pull/12751) — **open** and a competing
|
||||
less-exact alternative to #12987; its described multi-client non-convergence
|
||||
must not become the final model.
|
||||
- [PR #12339 — stop remote workspace pulls from resurrecting closed
|
||||
tabs](https://github.com/stablyai/orca/pull/12339) — **open**. Direct topology
|
||||
reconciliation context for explicit close and stale snapshots.
|
||||
- [PR #13013 — stop stale snapshot applies from erasing fresh terminal
|
||||
tabs](https://github.com/stablyai/orca/pull/13013) — **open**. Relevant legacy
|
||||
snapshot race; decide whether its machinery is deleted by the final authority
|
||||
cutover rather than layering another permanent reconciler.
|
||||
- [PR #12903 — retry snapshot tabs whose paths resolve after
|
||||
apply](https://github.com/stablyai/orca/pull/12903) — **open**. Deferred
|
||||
hydration is relevant legacy-importer context and may become deletable after
|
||||
authoritative namespace resolution; do not retain it by default.
|
||||
|
||||
## Delivery, backpressure, and lifecycle alternatives
|
||||
|
||||
- [Draft PR #12220 — keep SSH alive when a terminal outruns a slow
|
||||
link](https://github.com/stablyai/orca/pull/12220) — **open draft**. Useful G3
|
||||
and performance evidence, but its own description has behavioral blockers and
|
||||
a stalled-consumer wedge. Do not treat it as merge-ready.
|
||||
- [PR #10744 — durable lost-worker archive and mixed-version SSH
|
||||
revive](https://github.com/stablyai/orca/pull/10744) — **open** and broad. It
|
||||
is major architectural context for exact close, outcomes, retirement, and
|
||||
mixed versions, with substantial overlap. Mine proof and semantics rather
|
||||
than stacking blindly.
|
||||
- [PR #11801 — prune stale remote tab
|
||||
ghosts](https://github.com/stablyai/orca/pull/11801) — **open**. Its snapshot
|
||||
omission plus timer policy is relevant but may conflict with the rule that
|
||||
absence/time alone cannot prove destructive cleanup.
|
||||
|
||||
## Mandatory issue-to-journey matrix
|
||||
|
||||
This table binds customer incidents to the thirteen journeys in
|
||||
[`goalposts.md`](./goalposts.md#thirteen-required-production-journeys). Closing
|
||||
an issue or merging a narrow PR does not remove its oracle. Each row must be red
|
||||
on an unfixed baseline and green on the final candidate, or the user must accept
|
||||
explicit evidence that the issue is unrelated.
|
||||
|
||||
| Open issue(s) | Journey(s) | Mandatory discriminating outcome |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| [#11729](https://github.com/stablyai/orca/issues/11729) | 6, 9, 13 | Rocky Linux reconnect preserves the same workspace, transcript, pane, binding, PID, and environment without PTY growth. |
|
||||
| [#9819](https://github.com/stablyai/orca/issues/9819), [#9034](https://github.com/stablyai/orca/issues/9034) | 6, 13 | Reconnect cannot accumulate detached PTYs/session slots; explicit close and teardown reclaim the exact process, including lost-response retry. |
|
||||
| [#11006](https://github.com/stablyai/orca/issues/11006), [#12448](https://github.com/stablyai/orca/issues/12448), [#12699](https://github.com/stablyai/orca/issues/12699) | 2, 6 | Disconnect, source recovery, and wake keep the same live PID/session and never start a duplicate resume without proof of death. |
|
||||
| [#12447](https://github.com/stablyai/orca/issues/12447), [#11904](https://github.com/stablyai/orca/issues/11904) | 2, 6 | Durable close intent survives lost RPC, app/daemon/relay restart, retries exact retirement, reclaims the process, and never resurrects UI. |
|
||||
| [#10208](https://github.com/stablyai/orca/issues/10208) | 1, 9 | Worktree switch/restore preserves one exact pane/process and does not duplicate a tab. |
|
||||
| [#8585](https://github.com/stablyai/orca/issues/8585) | 2, 6, 12 | Cleanup affects only the proven relay generation across reconnect and version change; ID/path reuse cannot kill a successor. |
|
||||
| [#9138](https://github.com/stablyai/orca/issues/9138), [#11342](https://github.com/stablyai/orca/issues/11342) | 2, 12, 13 | Upgrade/restart converges old daemon generations and their PTYs without invisible sessions, leaks, or destructive guessing. |
|
||||
| [#8275](https://github.com/stablyai/orca/issues/8275) | 1, 2, 5 | Worktree teardown cannot terminate a sibling namespace or split-pane session. |
|
||||
| [#10415](https://github.com/stablyai/orca/issues/10415) | 1, 2, 5, 12 | Native Windows handles an unkillable PTY and protocol bump without daemon crash, live-session loss, or cross-session damage. |
|
||||
| [#9827](https://github.com/stablyai/orca/issues/9827), [#11339](https://github.com/stablyai/orca/issues/11339) | 2, 9 | Physical WSL restart restores the same shell command and agent session under the exact namespace/binding. |
|
||||
| [#12568](https://github.com/stablyai/orca/issues/12568) | 3, 6, 12 | Relay recovery/update lazily rediscovers the same SSH host/provider and restores only its exact sessions. |
|
||||
| [#11495](https://github.com/stablyai/orca/issues/11495), [#11574](https://github.com/stablyai/orca/issues/11574) | 4, 8, 10, 12, 13 | Paired restart/re-pair preserves stable host identity and every reported live host PTY at scale across independent updates. |
|
||||
| [#11265](https://github.com/stablyai/orca/issues/11265), [#9092](https://github.com/stablyai/orca/issues/9092) | 8, 13 | Live-but-stalled transport and sleep/wake automatically resume bounded delivery without an unrelated RPC kick or identity replacement. |
|
||||
| [#11803](https://github.com/stablyai/orca/issues/11803), [#9585](https://github.com/stablyai/orca/issues/9585), [#11800](https://github.com/stablyai/orca/issues/11800) | 3, 8 | Remote stop/host topology outcome remains durably retired after restart; no stale-owner tab or null-PTY ghost returns. |
|
||||
| [#9562](https://github.com/stablyai/orca/issues/9562), [#8129](https://github.com/stablyai/orca/issues/8129) | 3, 8, 13 | Complete boundary plus replay restores disconnected/restarted consumers without an empty session or lost semantic outcome. |
|
||||
| [#12241](https://github.com/stablyai/orca/issues/12241) | 8, 11, 13 | Paired partitions and retained-consumer state remain bounded; lost-consumer retirement safely restores compaction liveness. |
|
||||
| [#12683](https://github.com/stablyai/orca/issues/12683) | 8, 12 | Stale same-handle reconnect is generation-fenced and automatic recovery remains live across independent peer versions. |
|
||||
| [#10385](https://github.com/stablyai/orca/issues/10385), [#12140](https://github.com/stablyai/orca/issues/12140) | 8, 10, 12 | False-connected or rejected E2EE/pairing state fails explicitly and reauthenticates without stale identity, capability, or session adoption. |
|
||||
|
||||
Journeys 4, 5, 7, 10, and 11 remain mandatory even where no single issue fully
|
||||
specifies them. This matrix adds incident oracles; it never narrows the journey
|
||||
definitions.
|
||||
|
||||
## Original containment/refactor line
|
||||
|
||||
- [PR #12600 — break Activity and terminal React 185
|
||||
loops](https://github.com/stablyai/orca/pull/12600) — **merged**. It is current
|
||||
containment/background context, not the whole program and not a release gate.
|
||||
- [PR #12634 — replace React 185 timers with structural
|
||||
handoffs](https://github.com/stablyai/orca/pull/12634) — **open**. This is the
|
||||
direct structural follow-up for deleting #12600's window/timer brakes and must
|
||||
be reconciled with the broader terminal-session model.
|
||||
- [PR #13110 — sandboxed preload artifact
|
||||
guard](https://github.com/stablyai/orca/pull/13110) — **open**, independent of
|
||||
#13111, green at its head, no approval. Its unresolved major review says raw
|
||||
source matching can reject safe strings/comments; resolve or supersede that
|
||||
[finding](https://github.com/stablyai/orca/pull/13110#discussion_r3739646949)
|
||||
before calling it ready.
|
||||
|
||||
## Already merged base context
|
||||
|
||||
These were described as open in older notes but were merged when this handoff
|
||||
was written:
|
||||
|
||||
- [PR #12474 — isolate same-path folder workspace PTY
|
||||
identity](https://github.com/stablyai/orca/pull/12474)
|
||||
- [PR #12477 — stop a closed remote workspace window reading
|
||||
Ready](https://github.com/stablyai/orca/pull/12477)
|
||||
- [PR #12600 — React/terminal loop containment](https://github.com/stablyai/orca/pull/12600)
|
||||
|
||||
Refresh `main` and recalculate conflicts; do not follow the stale sequence
|
||||
`#12882 → #12474 → #12477 → #13111` as written.
|
||||
|
||||
## Required GitHub refresh before action
|
||||
|
||||
At minimum, re-query:
|
||||
|
||||
```bash
|
||||
gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url
|
||||
gh pr view 13110 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url
|
||||
gh issue view 11729 --repo stablyai/orca --json state,title,updatedAt,url
|
||||
```
|
||||
|
||||
Refresh the complete catalog in one batched request:
|
||||
|
||||
```bash
|
||||
handoff_query_text="$(tr '\n' ' ' < docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql)"
|
||||
gh api graphql -f query="$handoff_query_text"
|
||||
```
|
||||
|
||||
The query includes current PR state/check summaries and unresolved-thread state
|
||||
for #13110/#13111. Read PR bodies/diffs only for the slice being changed. Do not
|
||||
close, comment, merge, push, or checkout a PR from the handoff audit without
|
||||
explicit authorization.
|
||||
File diff suppressed because one or more lines are too long
@@ -1,275 +0,0 @@
|
||||
# Resume plan and validation record
|
||||
|
||||
This plan preserves the comprehensive objective while allowing independently
|
||||
safe stacks. It does not require reviving the 814-path construction snapshot.
|
||||
|
||||
## First safe commands
|
||||
|
||||
Run from the primary worktree before editing:
|
||||
|
||||
```bash
|
||||
pwd
|
||||
git status --short --branch --untracked-files=all
|
||||
git rev-parse HEAD origin/main
|
||||
git rev-list --left-right --count HEAD...origin/main
|
||||
git branch --list 'nwparker/react185-authority-snapshot'
|
||||
git show --stat --oneline fddb19f6977
|
||||
gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url
|
||||
```
|
||||
|
||||
Expected handoff checkpoint before other changes:
|
||||
|
||||
- branch `nwparker/sta-3077-reattach-pane-cardinality`;
|
||||
- HEAD `5369479be2953f45cea9ab5cfcde756dd9660548`;
|
||||
- snapshot branch and commit present;
|
||||
- only this handoff folder and its `.gitignore` allow-list are new handoff work.
|
||||
|
||||
If the branch, HEAD, or dirty paths differ, inventory and preserve them before
|
||||
continuing. Never reset, clean, overwrite, or switch away from unknown user work.
|
||||
|
||||
## Phase 0 — Reconcile reality before design or code
|
||||
|
||||
1. Refresh `origin/main` with `git fetch origin main`—never `git pull`—and
|
||||
refresh the live state/reviews of every PR in
|
||||
[`related-open-work.md`](./related-open-work.md) that touches the intended
|
||||
slice.
|
||||
2. Recalculate the candidate merge base, categorized LOC, changed-file overlap,
|
||||
and failing/obsolete tests.
|
||||
3. Freeze the program accounting baseline before implementation. The default is
|
||||
`5ed45739e94bdf6460364e033bfcec9b32c0b42a` (the base of #12600). Build an
|
||||
aggregate carry-in ledger for every program-attributable PR already merged
|
||||
after it; do not measure only the last PR in a stack.
|
||||
4. Read the final diff of #13111, #12264, and #12743 with `gh pr view` and
|
||||
`gh pr diff`, or in disposable isolated worktrees. Never use `gh pr checkout`
|
||||
or switch branches in the primary worktree. Select or design one coherent
|
||||
lease/reattach model; do not combine mutually inconsistent quarantine, kill,
|
||||
and non-kill policies.
|
||||
5. Reconcile #12882 before overlapping daemon-endpoint integration and #12760
|
||||
before overlapping relay-generation cleanup. Neither blocks unrelated SSH
|
||||
proof work by declaration.
|
||||
6. Treat the preserved snapshot as a local-only mine, not a branch to finish.
|
||||
Inspect it with `git show fddb19f6977:<path>` or in a disposable isolated
|
||||
worktree; never switch the primary worktree to it.
|
||||
|
||||
Decision output required before implementation:
|
||||
|
||||
- exact incident mechanisms being eliminated;
|
||||
- exact authority/identity semantics retained or changed;
|
||||
- explicit remote-shell lifecycle and user recovery behavior;
|
||||
- explicit lost-consumer liveness/compaction behavior;
|
||||
- PR/stack boundaries and dependency order;
|
||||
- production modules expected to be deleted; and
|
||||
- tests that will discriminate the baseline from the candidate; and
|
||||
- a predeclared performance protocol with fixed workloads, samples,
|
||||
statistical/noise bounds, deterministic counters, and raw-data locations.
|
||||
|
||||
## Phase 1 — Build discriminating journeys first
|
||||
|
||||
Before relying on a code change, create or repair oracles that reproduce the
|
||||
failure on an unfixed baseline.
|
||||
|
||||
### STA-3077 Docker SSH oracle
|
||||
|
||||
The Docker journey must:
|
||||
|
||||
- run a real OpenSSH server with `MaxSessions=1` verified from the server;
|
||||
- create a known pane and record tab, leaf, binding, relay PTY, and remote PID;
|
||||
- seed or naturally reproduce duplicate/stale lease state from the field;
|
||||
- exercise disconnect, client restart, reconnect, and authority/relay restart;
|
||||
- wait an explicit settle interval after successful reattach;
|
||||
- recensus visible panes, tabs, exact bindings, persisted leases, relay PTYs,
|
||||
and actual remote processes/PIDs;
|
||||
- prove no pane or shell is created, rebound, lost, or silently killed;
|
||||
- prove explicit unresolved recovery for state that cannot be established; and
|
||||
- exercise explicit close, lost shutdown response, reconnect retry, and
|
||||
worktree teardown; prove exact remote PID/session-slot reclamation while an
|
||||
uncertain sibling survives; and
|
||||
- fail for the intended reason on the unfixed baseline.
|
||||
|
||||
A clean reconnect that passes on both revisions is not this oracle.
|
||||
|
||||
### Duplicate-agent oracle family
|
||||
|
||||
Cover transient SSH loss, `restoreRequired`, sleeping-agent wake, worktree
|
||||
switch, and closed-tab restart. Assert process count and transcript/session
|
||||
ownership, not just pane/store state. No path may start a second `--resume`
|
||||
process while the first process is live or unknown.
|
||||
|
||||
### Daemon and remote oracles
|
||||
|
||||
Cover unknown inventory, endpoint handover, daemon restart, relay generation
|
||||
reuse, client sleep/wake, remote-runtime restart, and stale subscription repair.
|
||||
A retry limit may end an attempt or surface a failure; it may not itself prove a
|
||||
session dead or authorize destructive replacement.
|
||||
|
||||
## Phase 2 — Reconcile and approve the long-term design
|
||||
|
||||
The implementation design must explicitly settle:
|
||||
|
||||
- final-host identity versus client routing aliases;
|
||||
- canonical host-local namespace for worktree, folder, floating, drive, and UNC
|
||||
forms;
|
||||
- structural one-active-binding/lease ownership rather than optional metadata
|
||||
plus repair scans;
|
||||
- exact operation fencing across every transport;
|
||||
- explicit detach, close, retirement, orphan recovery, and process-reclamation
|
||||
operations;
|
||||
- durable outcome/replay semantics across app, renderer, host, and transport
|
||||
restarts;
|
||||
- mixed-version capability and isolated legacy behavior;
|
||||
- lost-device consumer retirement and compaction liveness;
|
||||
- one-way migration/cutover; and
|
||||
- the four-role deletion oracle and minimised, justified production-code target.
|
||||
|
||||
Reuse existing pane, PTY, incarnation, CAS, and three-valued liveness primitives
|
||||
where their semantics match. Do not equate client routing IDs with host identity
|
||||
or namespace without proving the full semantics.
|
||||
|
||||
If the chosen design differs from the preserved normative design, write an
|
||||
explicit mapping for every invariant and every journey, identify what changes,
|
||||
show equal or stronger safety/liveness, and obtain the user's approval. A
|
||||
handoff author or implementation agent cannot approve that scope change alone.
|
||||
|
||||
## Phase 3 — Implement as a convergent stack
|
||||
|
||||
Stacks are allowed for reviewability. Temporary production stopgaps are not the
|
||||
end state.
|
||||
|
||||
A sensible dependency shape is:
|
||||
|
||||
1. discriminating journey harnesses and behavior oracles;
|
||||
2. final shared identity, liveness, and exact-binding primitives;
|
||||
3. local/daemon/WSL/SSH/relay/paired/remote adapters using those primitives;
|
||||
4. durable outcome, projection, reset, and recovery lifecycle;
|
||||
5. one-way migration and capability cutover;
|
||||
6. deletion of superseded writers, ledgers, repair scans, timers used as verdicts,
|
||||
and duplicate state machines; retain `terminal-input-quarantine.ts` until a
|
||||
replacement passes its destructive-input oracle; and
|
||||
7. full platform, skew, performance, packaging, and independent review proof.
|
||||
|
||||
Each stack layer must be safe and tested, but the comprehensive status remains
|
||||
0/8 until the final integrated candidate satisfies the complete gates. Do not
|
||||
advertise the authoritative capability or claim program completion mid-stack.
|
||||
|
||||
## Phase 4 — Run the complete proof matrix
|
||||
|
||||
Use [`goalposts.md`](./goalposts.md) as the checklist. All thirteen production
|
||||
journeys and every row in the mandatory
|
||||
[`issue-to-journey matrix`](./related-open-work.md#mandatory-issue-to-journey-matrix)
|
||||
are required. At minimum the final matrix must include:
|
||||
|
||||
- real macOS, Linux floor, Windows, and WSL;
|
||||
- local, daemon, direct SSH, nested SSH, paired runtime, and remote server;
|
||||
- one Docker host with `MaxSessions=1` and two simultaneous Docker hosts;
|
||||
- git worktree, folder, floating, drive, and UNC workspaces;
|
||||
- old/new peers in both directions at every changed app/daemon, SSH relay,
|
||||
paired-runtime, remote-server, and mobile/E2EE boundary;
|
||||
- app, renderer, daemon, relay, and remote-runtime restart;
|
||||
- crash cuts around durable operations and identity reset;
|
||||
- unknown/timeout/disconnect/lost-response/concurrent-replacement cases;
|
||||
- actual remote PID and process accounting; and
|
||||
- A/B performance and memory results under the predeclared protocol, including
|
||||
raw samples, confidence intervals, deterministic counters, and leak slopes.
|
||||
|
||||
## Phase 5 — Delete, census, rebase, and review
|
||||
|
||||
1. Trace every new production module from a real entrypoint.
|
||||
2. Delete unreachable modules and fixtures in production trees.
|
||||
3. Delete all superseded legacy writers and reconciliation state machines after
|
||||
cutover.
|
||||
4. Rebase onto the actual PR base.
|
||||
5. Rerun all journeys and static/package checks on the rebased SHA.
|
||||
6. Report production, tests, docs, CI/runner, generated, and vendored LOC
|
||||
separately.
|
||||
7. Minimise aggregate production source net LOC against the frozen pre-program
|
||||
baseline, including all program-attributable carry-in and stack layers, and
|
||||
justify every net addition as required by D1.
|
||||
8. Run an independent repository review and the release-readiness checklist.
|
||||
9. Resolve every P0–P2 finding.
|
||||
10. Only then prepare the comprehensive PR and ask the user to approve any
|
||||
deliberate design change.
|
||||
|
||||
## Validation status at this handoff
|
||||
|
||||
This was a documentation and read-only audit turn. No product test command was
|
||||
rerun, so this handoff claims no fresh local product-test pass.
|
||||
|
||||
Exact validation commands run for this handoff:
|
||||
|
||||
```bash
|
||||
git status --short --branch --untracked-files=all
|
||||
git rev-parse HEAD origin/main
|
||||
git rev-list --left-right --count HEAD...origin/main
|
||||
git merge-base HEAD origin/main
|
||||
git diff --numstat --no-renames "$(git merge-base HEAD origin/main)"..HEAD
|
||||
gh pr view 13111 --repo stablyai/orca --json state,isDraft,mergeable,mergeStateStatus,reviewDecision,statusCheckRollup,headRefOid,baseRefOid,url
|
||||
handoff_query_text="$(tr '\n' ' ' < docs/reference/terminal-session-correctness-handoff-2026-08-07/github-overlap-query.graphql)"
|
||||
gh api graphql -f query="$handoff_query_text"
|
||||
pnpm exec oxfmt --check docs/reference/terminal-session-correctness-handoff-2026-08-07/*.md docs/reference/terminal-session-correctness-handoff-2026-08-07/*.graphql
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Outcomes:
|
||||
|
||||
- repository commands exited 0 and produced the branch/SHA/distance recorded in
|
||||
`README.md`;
|
||||
- the current-HEAD numstat reproduced `+309` production, `+914` tests/E2E,
|
||||
`+146` docs, `+43` CI/config, and `+1,412` total net LOC;
|
||||
- the reproducible GraphQL catalog returned all 65 referenced items and one
|
||||
unresolved, non-outdated review thread on each of #13110 and #13111;
|
||||
- all 30 linked issues appear in the mandatory issue-to-journey matrix and the
|
||||
reusable GraphQL query;
|
||||
- #13111 was open, non-draft, mergeable/clean, with 47 successful and 5 skipped
|
||||
checks and no approval decision; and
|
||||
- formatting and whitespace checks exited 0 after documentation formatting.
|
||||
|
||||
Verified in this worktree:
|
||||
|
||||
- repository branch, HEAD, merge base, local `origin/main`, branch distance,
|
||||
and clean pre-documentation status;
|
||||
- snapshot branch/commit presence and the 814-path preservation claim;
|
||||
- current source implementation of the narrow #13111 behavior and gaps;
|
||||
- current categorized #13111 diff census;
|
||||
- original G0–G7, thirteen journeys, and PR proof contract; and
|
||||
- live GitHub PR/issue state through batched GraphQL queries.
|
||||
|
||||
Live GitHub verification at the time of writing showed:
|
||||
|
||||
- #13111: open, non-draft, mergeable/clean, checks successful, no approval, one
|
||||
unresolved major review thread;
|
||||
- #13110: open, non-draft, checks successful, no approval, one unresolved major
|
||||
review thread;
|
||||
- #12264 and #12743: open competing STA-3077 implementations;
|
||||
- #12474, #12477, and #12600: merged; and
|
||||
- the other items in `related-open-work.md`: open unless explicitly labeled
|
||||
merged or draft.
|
||||
|
||||
These states are ephemeral and must be refreshed.
|
||||
|
||||
Historical pause receipts from the 60,903-line construction snapshot remain
|
||||
construction evidence only. They do not prove the current branch, G0–G7, or any
|
||||
required journey.
|
||||
|
||||
## Stop conditions
|
||||
|
||||
Stop and return to the user before:
|
||||
|
||||
- narrowing or replacing a named goalpost;
|
||||
- accepting a correctness/performance tradeoff;
|
||||
- deciding that remote orphan shells may be killed or leaked permanently;
|
||||
- changing wire semantics without a compatibility design;
|
||||
- shipping a positive-production-LOC comprehensive stack;
|
||||
- merging, closing, commenting on, or force-updating external PRs; or
|
||||
- deleting preserved user work.
|
||||
|
||||
## Short takeover prompt
|
||||
|
||||
> Continue the comprehensive terminal-session correctness program from
|
||||
> [`README.md`](./README.md). Preserve the fixed status of **0/8 proven gates
|
||||
> and 0/13 proven journeys** until every proof in
|
||||
> [`goalposts.md`](./goalposts.md) passes on one rebased candidate. Reconcile the
|
||||
> conflicting open work in [`related-open-work.md`](./related-open-work.md),
|
||||
> build discriminating real SSH/daemon/paired/remote/cross-platform journeys,
|
||||
> remove root causes and superseded reconciliation, minimise and justify every
|
||||
> production-code addition per D1, and finish with no correctness or performance
|
||||
> regression.
|
||||
@@ -1,277 +0,0 @@
|
||||
# Terminal session correctness — test design
|
||||
|
||||
How we decide a test is worth trusting, what we built, and what we could not
|
||||
prove. Read [`design-overview.md`](./design-overview.md) first.
|
||||
|
||||
---
|
||||
|
||||
## 1. The problem with "the tests pass"
|
||||
|
||||
A green test proves nothing on its own. It might pass because the code is
|
||||
correct, or because the test never looked.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
T["Test is green"] --> Q{"Would it go red if<br/>we broke the code?"}
|
||||
Q -->|"Yes"| G["Real evidence"]
|
||||
Q -->|"No"| W["Worthless.<br/>Green either way."]
|
||||
Q -->|"Never checked"| U["Unknown.<br/>Treat as worthless."]
|
||||
|
||||
style G fill:#e6ffe6,stroke:#00aa00
|
||||
style W fill:#ffcccc,stroke:#cc0000
|
||||
style U fill:#fff4e6,stroke:#dd8800
|
||||
```
|
||||
|
||||
This is not hypothetical. **It happened to us twice:**
|
||||
|
||||
- An end-to-end test was reported as proving the reconnect fix. It passed with
|
||||
the fix **and** with the fix removed. Retracted.
|
||||
- A guard was added, every test stayed green, and the guard had **no production
|
||||
caller at all**. The capability existed; nothing used it. Tests passed the
|
||||
whole time because they called the capability directly.
|
||||
|
||||
---
|
||||
|
||||
## 2. The rule we adopted: the four-step proof
|
||||
|
||||
Nothing counts as evidence until it has been through this loop.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
S1["1. Run it<br/><b>green</b>"] --> S2["2. Delete the guard<br/>from production code"]
|
||||
S2 --> S3["3. Run it<br/><b>MUST go red</b>"]
|
||||
S3 --> S4["4. Put the guard back<br/><b>green again</b>"]
|
||||
S3 -.->|"stays green"| F["Test is worthless.<br/>Say so. Don't ship it<br/>as proof."]
|
||||
|
||||
style S3 fill:#e6ffe6,stroke:#00aa00
|
||||
style F fill:#ffcccc,stroke:#cc0000
|
||||
```
|
||||
|
||||
Three rules that make this honest rather than theatre:
|
||||
|
||||
**Delete a real guard, not the test.** The mutation goes in _production_ code. If
|
||||
you tweak the test until it fails, you have proved nothing.
|
||||
|
||||
**Prefer one clause at a time.** If a test has three claims, a good mutation
|
||||
reddens exactly one and leaves the other two green. That proves the claims
|
||||
_independently_ rather than as a bundle.
|
||||
|
||||
**Watch it fail yourself.** A report saying "it went red" is not the same as
|
||||
seeing it. Of the twelve oracles here, six were personally re-verified, and two
|
||||
of those re-checks overturned the original claim.
|
||||
|
||||
### The trap that nearly fooled us twice
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
M["Apply mutation"] --> C{"Did the mutation<br/>actually land?"}
|
||||
C -->|"No — silent no-op"| X["Test stays green.<br/>You conclude 'no teeth'.<br/><b>Wrong conclusion.</b>"]
|
||||
C -->|"Yes"| R["Now the result<br/>means something"]
|
||||
|
||||
style X fill:#ffcccc,stroke:#cc0000
|
||||
```
|
||||
|
||||
Twice a search-and-replace matched the wrong line and changed nothing. The test
|
||||
stayed green and looked like a failed oracle. **Always verify the mutation landed
|
||||
before believing the result.**
|
||||
|
||||
---
|
||||
|
||||
## 3. The three layers of test
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph L1["Unit — milliseconds"]
|
||||
U1["Pure logic.<br/>Does the rule compute<br/>the right answer?"]
|
||||
end
|
||||
subgraph L2["Integration — seconds"]
|
||||
I1["Real Electron app.<br/>Real restarts.<br/>Real shell processes."]
|
||||
end
|
||||
subgraph L3["End-to-end — minutes"]
|
||||
E1["Real Docker container<br/>running real OpenSSH.<br/>Real network faults."]
|
||||
end
|
||||
L1 --> L2 --> L3
|
||||
L1 -.->|"fast, narrow"| N1[" "]
|
||||
L3 -.->|"slow, real"| N2[" "]
|
||||
style N1 fill:none,stroke:none
|
||||
style N2 fill:none,stroke:none
|
||||
```
|
||||
|
||||
The rule of thumb: **assert the strongest thing you can actually observe.**
|
||||
|
||||
---
|
||||
|
||||
## 4. Why "same id" is not good enough
|
||||
|
||||
The most valuable lesson from this work.
|
||||
|
||||
A restart test naturally asks: _is it the same terminal?_ The easy check is
|
||||
whether the ids match. That check is **too weak**, and we can prove it.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
B["Break the quit path so it<br/>destroys shells instead of<br/>detaching from them"] --> R["Restart the app"]
|
||||
R --> C1["tab id: same ✅"]
|
||||
R --> C2["pane id: same ✅"]
|
||||
R --> C3["pty id: same ✅"]
|
||||
R --> C4["<b>OS process: DIFFERENT</b> ❌<br/>pid 13756 → 8852"]
|
||||
C3 --> W["A weak test passes here.<br/>Your shell is gone."]
|
||||
C4 --> S["Our test fails here."]
|
||||
|
||||
style W fill:#ffcccc,stroke:#cc0000
|
||||
style S fill:#e6ffe6,stroke:#00aa00
|
||||
```
|
||||
|
||||
So the test asks the shell to report its **own process id** through the real
|
||||
input path, then reads that process's **kernel start time**. Same id _and_ same
|
||||
start time means the same process really survived — a recycled pid cannot pass.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant T as Test
|
||||
participant A as Orca
|
||||
participant S as Shell
|
||||
participant K as Kernel
|
||||
T->>A: type "echo MARKER=$$"
|
||||
A->>S: (through the real write path)
|
||||
S-->>T: MARKER=13756
|
||||
T->>K: start time of 13756?
|
||||
K-->>T: 02:01:29.337
|
||||
Note over T: restart the app
|
||||
T->>S: ask again
|
||||
S-->>T: MARKER=13756
|
||||
T->>K: start time of 13756?
|
||||
K-->>T: 02:01:29.337 ✅ same process
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. What is covered
|
||||
|
||||
### Journey 1 — a terminal survives restart · **PROVEN**
|
||||
|
||||
Two claims: the same pane, binding and **OS process** survive a reload and an app
|
||||
restart; and a stale operation is refused.
|
||||
|
||||
| Platform | Result | Mutation A<br/>(destructive quit) | Mutation B<br/>(remove stale-write guard) |
|
||||
| ------------------ | ------ | --------------------------------- | ----------------------------------------- |
|
||||
| macOS | 2 pass | red | red |
|
||||
| Linux, native | 2 pass | red | red |
|
||||
| Windows 11, native | 2 pass | **red — only claim 1** | **red — only claim 2** |
|
||||
|
||||
The two mutations hit **different claims**, which is what proves them separately.
|
||||
|
||||
### Journey 2 — daemon and WSL · **PROVEN**
|
||||
|
||||
| Environment | Result | Mutation A<br/>(liveness can't say "unknown") | Mutation B<br/>(widen owner fallback) |
|
||||
| --------------- | ------ | --------------------------------------------- | ------------------------------------- |
|
||||
| macOS | 3 pass | red — claim 1 only | red — claim 3 only |
|
||||
| Linux, native | 3 pass | red — claim 1 only | red — claim 3 only |
|
||||
| WSL2 on Windows | 3 pass | red — claim 1 only | — |
|
||||
|
||||
> **A subtlety worth knowing.** These tests run in order, so when the first one
|
||||
> fails the others report "did not run" — that is a _skip_, not a pass. To prove
|
||||
> the mutation really only hit claim 1, we re-ran claims 2 and 3 **alone, with
|
||||
> the mutation still applied**, and watched them stay green.
|
||||
|
||||
### Other oracles that bite
|
||||
|
||||
| Oracle | Mutation that reddens it |
|
||||
| ---------------------------- | ------------------------------------------------ |
|
||||
| Reconnect grafts no pane | Remove "bind, never create" |
|
||||
| Superseded-keystroke fence | Remove the guard from the handlers |
|
||||
| Guard is actually wired up | Remove the call sites, keep the function |
|
||||
| Workspace ids don't collide | Restore the suffix-stripping comparison |
|
||||
| Wire compatibility | Restore the wrong "expired" message |
|
||||
| `rm -rf` hazard stays closed | Disable the quarantine → `cho hi; rm -rf x` runs |
|
||||
|
||||
That last one is why we did **not** delete the quarantine module.
|
||||
|
||||
---
|
||||
|
||||
## 6. What we could not prove, and why
|
||||
|
||||
This is the honest half.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
P["Property we want<br/>to prove"] --> Q{"Is there a shipped<br/>guard protecting it?"}
|
||||
Q -->|"Yes"| M["Remove it → test reddens.<br/><b>Provable</b>"]
|
||||
Q -->|"No — it's true<br/>by construction"| C["Nothing to remove.<br/><b>Not provable by mutation</b>"]
|
||||
Q -->|"No code exists<br/>at all"| N["Nothing to test.<br/><b>Not writable</b>"]
|
||||
|
||||
style M fill:#e6ffe6,stroke:#00aa00
|
||||
style C fill:#fff4e6,stroke:#dd8800
|
||||
style N fill:#ffcccc,stroke:#cc0000
|
||||
```
|
||||
|
||||
| Journey | Why not proven |
|
||||
| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Two hosts don't interfere | True **by construction** — each host has its own channel, so there is no shared thing to break. Four attempts to redden it failed, correctly. |
|
||||
| Lazy host discovery | One claim has no mutation that hits it alone; removing the real guard breaks setup before the claim is reached. |
|
||||
| `MaxSessions=1` reconnect | Two claims redden. A third survived **four** different guard removals — nothing shipped protects it. |
|
||||
| Mixed versions, live | The new message **never crosses a version boundary** in production. Our in-process tests exercise a route the real app doesn't take. Withdrawn. |
|
||||
| Performance | 1 of 10 required dimensions measured. The keystroke guard costs ~14ns, but measured outside the real app. |
|
||||
| Three others | The features they test **do not exist** in the code. No test can be written. |
|
||||
|
||||
**"True by construction" is a good outcome, not a failure** — it means the
|
||||
property holds because of how the code is shaped, not because a guard is
|
||||
watching. It just can't be demonstrated by breaking something.
|
||||
|
||||
---
|
||||
|
||||
## 7. Environment traps that can fake a result
|
||||
|
||||
Real problems we hit, worth knowing before trusting any run.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph T1["Shared temp directory"]
|
||||
A1["Test harness writes a pointer<br/>to a machine-wide temp path"] --> A2["Two runs at once<br/>overwrite each other"]
|
||||
A2 --> A3["Can fake a red<br/><b>and</b> hide a real one"]
|
||||
end
|
||||
subgraph T2["Stale build"]
|
||||
B1["e2e runs a built app"] --> B2["Edit source, forget rebuild"]
|
||||
B2 --> B3["You tested the old code"]
|
||||
end
|
||||
style A3 fill:#ffcccc,stroke:#cc0000
|
||||
style B3 fill:#ffcccc,stroke:#cc0000
|
||||
```
|
||||
|
||||
So every run isolates its temp directory, and we check build timestamps against
|
||||
source edits before believing a result.
|
||||
|
||||
Two more, found by running on machines we don't develop on:
|
||||
|
||||
- A test failed on **every** Linux run. Not a product bug: it read a value once,
|
||||
immediately after a reload, where its sibling helper polls for 15 seconds.
|
||||
- A test could not run on **Windows at all** — it used `echo $$` and `ps`, which
|
||||
don't exist there. The Windows claim wasn't unproven, it was _unprovable_.
|
||||
|
||||
Both were bugs in tests we wrote, found only by running on the real platform.
|
||||
|
||||
---
|
||||
|
||||
## 8. Scoreboard
|
||||
|
||||
```mermaid
|
||||
pie showData
|
||||
title Journeys
|
||||
"Proven with discrimination" : 2
|
||||
"Oracle exists, cannot discriminate" : 5
|
||||
"No code exists to test" : 3
|
||||
"Not yet attempted" : 3
|
||||
```
|
||||
|
||||
| Measure | Value |
|
||||
| ------------------------------- | ----------- |
|
||||
| Journeys proven | **2 of 13** |
|
||||
| Gates proven | **0 of 8** |
|
||||
| Oracles that bite | 12 |
|
||||
| …personally re-verified | 6 |
|
||||
| Claims retracted after checking | 2 |
|
||||
| Production code added | +214 lines |
|
||||
|
||||
The retraction count is the number to look at. Two claims that looked green were
|
||||
withdrawn after checking. That is the process working — and the reason to trust
|
||||
the two journeys that survived it.
|
||||
@@ -26,7 +26,6 @@
|
||||
"prepare": "husky",
|
||||
"test": "node config/scripts/ensure-native-runtime.mjs --runtime=node && vitest run --config config/vitest.config.ts",
|
||||
"test:repro:remote-agent-session": "pnpm run build:cli && pnpm run build:electron-vite && node config/scripts/remote-agent-session-authority-repro.mjs",
|
||||
"test:repro:daemon-replacement-live-agent-pty-preservation": "pnpm run build:electron-vite && node config/scripts/daemon-replacement-live-agent-pty-preservation-repro.mjs",
|
||||
"check:reliability-gates": "node config/scripts/check-reliability-gates.mjs",
|
||||
"check:max-lines-ratchet": "node config/scripts/check-max-lines-ratchet.mjs",
|
||||
"check:feature-wall-assets": "node config/scripts/check-feature-wall-assets.mjs",
|
||||
@@ -103,9 +102,6 @@
|
||||
"test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs",
|
||||
"test:e2e:ssh-docker-watcher-isolation": "node config/scripts/run-ssh-docker-watcher-isolation-e2e.mjs",
|
||||
"test:e2e:ssh-docker-terminal-parking": "node config/scripts/run-ssh-docker-terminal-parking-e2e.mjs",
|
||||
"test:e2e:ssh-docker-reconnect-pane-cardinality": "node config/scripts/run-ssh-docker-reconnect-pane-cardinality-e2e.mjs",
|
||||
"test:e2e:ssh-docker-maxsessions-identity": "node config/scripts/run-ssh-docker-maxsessions-identity-e2e.mjs",
|
||||
"test:e2e:ssh-docker-two-hosts-isolation": "node config/scripts/run-ssh-docker-two-hosts-isolation-e2e.mjs",
|
||||
"test:e2e:nested-runtime-ssh": "node config/scripts/run-nested-runtime-ssh-e2e.mjs",
|
||||
"test:e2e:source-control-scale": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/source-control-large-file-count.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"win-update-e2e": "node tests/tools/win-update-e2e/run.mjs",
|
||||
@@ -131,7 +127,6 @@
|
||||
"bench:cold-park-resource": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs",
|
||||
"bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs",
|
||||
"test:e2e:remote-bulk-open-freeze": "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:daemon-session-liveness": "pnpm run ensure:electron-runtime && pnpm exec playwright test --config tests/playwright.daemon-liveness.local.config.ts",
|
||||
"test:e2e:ssh-docker-bulk-open-freeze": "node config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs",
|
||||
"repro:live-remote-bulk-open-freeze": "node config/scripts/live-remote-bulk-open-freeze-repro.mjs",
|
||||
"repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs"
|
||||
|
||||
@@ -94,7 +94,9 @@ describe('OpenCode source discovery with a stalled WSL data directory', () => {
|
||||
// silent [] here reads as "no OpenCode sessions" on a clean scan.
|
||||
await expect(discoveries).resolves.toHaveLength(1)
|
||||
expect(
|
||||
issues.some((issue) => issue.agent === 'opencode' && issue.path === `${WSL_HOME}/opencode`)
|
||||
issues.some(
|
||||
(issue) => issue.agent === 'opencode' && issue.path === `${WSL_HOME}/opencode`
|
||||
)
|
||||
).toBe(true)
|
||||
} finally {
|
||||
restoreEnv('XDG_DATA_HOME', previousXdg)
|
||||
|
||||
@@ -26,139 +26,6 @@ hands → probe once more → `rename` in one syscall → verify we kept it.
|
||||
- **Never collapse "can't tell" into "dead."** Only `connected` means occupied; only
|
||||
`refused`/`missing` prove death. A timeout or `EPERM` proves nothing and must decline — treating
|
||||
it as death deletes an endpoint still serving every terminal on the host.
|
||||
- **"Can't tell" does not license a kill at launch either.** When the launcher cannot establish
|
||||
what a health-check-failing daemon is hosting, it holds it in degraded mode rather than
|
||||
replacing it. Only the daemon itself can prove it is empty, over IPC; the process table may
|
||||
only ever *raise* a verdict toward "occupied", never lower one toward a kill.
|
||||
|
||||
Two exclusions apply **to that residual only** — not to a daemon already proven occupied:
|
||||
an endpoint that is proven dead (a cold start has nothing to hold), and `rejected` (it
|
||||
answered and refused, so it can never be adopted and its sessions can never be reattached).
|
||||
|
||||
A `rejected` daemon that process evidence shows *is* hosting live PTYs is still held, because
|
||||
the choice there is between unreachable-but-running agents and dead ones. Restart recovers it
|
||||
at the documented cost.
|
||||
|
||||
The cost is deliberate and known: a wedged-but-empty daemon is no longer replaced at launch,
|
||||
so #8689 degrades to "restart it from Manage Sessions" instead of being handled automatically.
|
||||
And an endpoint held by something that accepts connections but never speaks the protocol — a
|
||||
foreign process, or our own permanently wedged daemon — reads as an incumbent on *every*
|
||||
launch, so it stays degraded with no auto-recovery. `killStaleDaemon` only kills a process
|
||||
whose identity matches the pid record, so Restart cannot clear that one; the degraded message
|
||||
says so and points at quit-and-relaunch.
|
||||
|
||||
Two pieces exist only to keep that cost from growing, and both have been proposed for deletion
|
||||
on the reasoning that "'unknown' and 'occupied' now behave the same". They do not. The process-
|
||||
table evidence read is what holds a daemon whose socket entry vanished while it still hosts
|
||||
agents — the occupied branch has no proven-dead check and the unknown hold does. And the
|
||||
grace-retry loop is worth *more* since the hold landed, because a counted `occupied` reaches
|
||||
full adoption where the alternative is a degraded hold.
|
||||
That was chosen over the alternative, which was killing daemons whose live agents we had
|
||||
merely failed to observe — unrecoverable, versus one click.
|
||||
|
||||
Three paths still reach a kill, and each is a residual rather than a guarantee. Adversarial
|
||||
review named all three; none is a regression against the pre-hold behaviour, and none should
|
||||
be closed by weakening the rules above.
|
||||
|
||||
- **`unknown` + a proven-dead endpoint, when process evidence is unavailable.** The endpoint
|
||||
probe proves the *entry* is gone, not the *process*; a socket entry can vanish while the
|
||||
daemon still hosts agents. Evidence covers that on POSIX — it runs for any `unknown`, not
|
||||
only a live endpoint — so the gap is where evidence cannot answer: the clock is spent, the
|
||||
pid will not verify, or `ps` is blind. Not reachable on Windows, where a named pipe vanishes
|
||||
with its process, so a dead endpoint there implies a dead daemon and no agents to lose.
|
||||
- **`unknown` + `rejected`, when evidence is unavailable.** "Cannot be adopted" is not the
|
||||
same as "cannot be preserved": its agents keep running even though nothing can ever reattach
|
||||
to them. Killing is chosen deliberately, because a daemon that can never be adopted and is
|
||||
never replaced leaves the app permanently degraded with no route back. Reconsider only with
|
||||
a way for the user to choose.
|
||||
- **TOCTOU between the verdict and the kill.** The right fix is known and was implemented and
|
||||
reverted once, deliberately: ask the daemon to retire itself via the existing `shutdownIfIdle`
|
||||
RPC immediately before the kill, and treat only its own `{retiring: true}` as permission.
|
||||
The daemon answers that atomically — sole authenticated client, nothing being created or
|
||||
attached, zero sessions — and closes its listener before acknowledging, so nothing can slip
|
||||
in behind the proof. A second `listSessions` would only move the race.
|
||||
|
||||
It was reverted because it makes every empty-verdict replacement depend on a new round trip,
|
||||
and any failure of that round trip must mean hold — which turns a rare race into a new,
|
||||
common failure mode, and worsens #8689 whenever the call is merely slow. It also changed the
|
||||
behaviour of two endpoint-identity tests in ways that were not quickly explainable. Land it
|
||||
on a green base with its own review, not as an addendum.
|
||||
An `empty` answer can go stale — another Orca
|
||||
instance may create a session before the ladder runs — and a dead endpoint can be
|
||||
republished. Nothing revalidates immediately before the kill, and `liveOwnerSurvived` is
|
||||
read only afterwards. Pre-existing, and narrowed by this change rather than widened: the
|
||||
window now opens only after the daemon has itself reported zero sessions.
|
||||
|
||||
Known limits of the process-table evidence, none of which can license a kill on their own —
|
||||
each only fails to *raise* a verdict, so the cost is a hold not taken:
|
||||
|
||||
- A PTY whose session leader has exited leaves its still-running child reparented outside the
|
||||
daemon's descendant tree. The walk cannot see it, so a daemon with real work can read as
|
||||
childless.
|
||||
- On Windows the evidence abstains entirely. A daemon that closed its listener but is still
|
||||
draining sessions therefore has no protection from the endpoint-dead path.
|
||||
- The self-spawned-probe exclusion matches an exact argv (`sh -c exit 0`). A hosted session
|
||||
leader whose executable basename is `sh` and whose command is exactly that would be
|
||||
discarded. Contrived — `exit 0` returns immediately — but it is executable identity the
|
||||
match cannot establish.
|
||||
|
||||
The owner check covers the operations that can destroy or corrupt a session — write, resize,
|
||||
shutdown, sendSignal, attach. It does **not** cover `pauseProducer`, `resumeProducer`,
|
||||
`setPtyBackgrounded`, `clearBuffer`, `closeStartupQueryAuthority`, `acknowledgeDataEvent`, or
|
||||
the per-session queries, which still route raw. For an unresolved daemon id those reach the
|
||||
fallback silently: a buffer clear reports success while the daemon's history survives, and
|
||||
flow control paces a producer that is not the one emitting. Pre-existing and unchanged here.
|
||||
|
||||
Before extending the check to them, note that `acknowledgeDataEvent` is called straight from
|
||||
an `ipcMain.on` listener and `setPtyBackgrounded` synchronously from a callback, neither with
|
||||
a boundary — so adding a throwing owner check without changing those call sites converts a
|
||||
silent misroute into an escaping exception.
|
||||
|
||||
**If you ever raise the classification budget, gate the replace path on headroom first.**
|
||||
The budget serves two verdicts with opposite time-costs: reaching "don't kill" slowly is free,
|
||||
because the daemon survives however long it took, while reaching `empty` slowly is not — the
|
||||
kill ladder (~11.5s) and the fork (~10s) still have to fit before the 60s fail-open. At 34s
|
||||
that case cannot arise (34 + 21.5 = 55.5). Raise the budget and it can, and an overrun there
|
||||
is the worst branch available: daemon killed, replacement forked and then discarded, no
|
||||
provider installed, Restart broken. The guard is to hold instead of replacing when the
|
||||
remaining headroom cannot fund the ladder and the fork — safe precisely because that path has
|
||||
proven the daemon empty, so holding costs no agents. Use `holdIncumbentDaemon()`, not
|
||||
`preserveDaemon()`, which opens a non-shared 20s handshake and could overrun the deadline it
|
||||
is meant to respect.
|
||||
|
||||
Why it is unreachable at 34s is structure, not margin, and the distinction is the point: the
|
||||
hold decoupled long classification from the replace path. A verdict of `empty` means the
|
||||
daemon *answered*, so it resolved fast by construction; `unknown` + proven-dead means nothing
|
||||
is listening, so the probe settles in ~500ms and the ladder short-circuits on ESRCH. The path
|
||||
that actually consumes the budget — a wedge that never answers — now ends in a hold, which
|
||||
pays neither the ladder nor the fork. The long path and the expensive tail are disjoint.
|
||||
Raising the budget is what re-couples them, by extending how late an `empty` may legally
|
||||
arrive (~22s in at 34s; ~32s in at 44s). The raise creates the case; it does not merely
|
||||
expose it.
|
||||
|
||||
Costing the guard honestly: the launcher closure does not receive the startup abort signal,
|
||||
but `createOutOfProcessLauncher` is a factory called from inside `initDaemonPtyProvider`,
|
||||
where `signal` is in scope. A third factory parameter closed over there leaves
|
||||
`DaemonLauncher`'s call signature — all `DaemonSpawner` knows about — unchanged. One
|
||||
parameter, not a spawner change. Record alongside it that a closed-over startup signal is
|
||||
meaningful only for the startup launch: `runRestartDaemon` reuses the same spawner and the
|
||||
`respawn` closure re-enters the same launcher, and both would read a signal that never
|
||||
aborts, because `servicesSettled` clears the fail-open timer once init succeeds. That is
|
||||
correct — later restarts are not under the startup gate — but it reads like a bug without
|
||||
the sentence.
|
||||
|
||||
Two things erode that margin rather than consume it, and neither is bounded by this budget:
|
||||
the `health === 'healthy'` branch never consults `classificationRemainingMs()` at all
|
||||
(`resolveOccupancyOverIpc` passes no `budgetMs`, so it takes the 19s default) and also ends
|
||||
in a cleanup and a fork; and packaged Windows follows the fork with a daemon-host directory
|
||||
copy of unbounded size. Both stay under today only because reaching them requires a verdict
|
||||
that arrives early.
|
||||
|
||||
**Do not try to fix this by tuning the classification budget.** Ten review rounds each found a
|
||||
different timing band where a bounded classification kills a session an unbounded one keeps.
|
||||
Matching the old tolerance for a single probe costs more clock than the 60s startup fail-open
|
||||
leaves once the kill ladder and the fork are paid for. The budget is a latency bound, not a
|
||||
correctness parameter, and it must stay that way.
|
||||
- **`link` first, never an unconditional `rename`.** `rename` replaces whatever it finds, so it
|
||||
would let a starting daemon destroy a healthy one. `link` fails loudly and forces the liveness
|
||||
question.
|
||||
|
||||
@@ -4,10 +4,8 @@ import {
|
||||
decodeDaemonResponseError,
|
||||
isDaemonEndpointGoneError,
|
||||
SessionNotFoundError,
|
||||
TerminalHostGoneError,
|
||||
TerminalSessionOwnerUnverifiedError
|
||||
TerminalHostGoneError
|
||||
} from './daemon-errors'
|
||||
import { isPtyWriteUnavailableError } from '../providers/pty-write-unavailable-error'
|
||||
import { mapRuntimeError } from '../runtime/rpc/errors'
|
||||
|
||||
function socketError(code: string, syscall: string): Error & { code: string; syscall: string } {
|
||||
@@ -68,27 +66,3 @@ describe('isDaemonEndpointGoneError', () => {
|
||||
expect(response.error).toEqual({ code: 'runtime_error', message: 'terminal_host_gone' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('TerminalSessionOwnerUnverifiedError classification', () => {
|
||||
const error = new TerminalSessionOwnerUnverifiedError('pty-1')
|
||||
|
||||
it('reads as an unavailable write, so a throw mid-paste reaches the renderer', () => {
|
||||
// Without this the remaining chunks are dropped with no pty:writeUnavailable, and the pane
|
||||
// never re-attaches — a silent truncation the user has no way to attribute.
|
||||
expect(isPtyWriteUnavailableError(error)).toBe(true)
|
||||
})
|
||||
|
||||
it('does not read as an already-gone session', () => {
|
||||
// The reason this is not a SessionNotFoundError: pty.ts's isPtyAlreadyGoneError matches
|
||||
// /Session not found/i and synthesizes an exit, which would report a session as dead
|
||||
// precisely when we could not establish that it was. Matching that predicate's shape here
|
||||
// rather than importing it, because it is private to the IPC layer.
|
||||
expect(/Session not found/i.test(error.message)).toBe(false)
|
||||
expect(/Session not found/i.test(new SessionNotFoundError('pty-1').message)).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps its own identity for callers that match on it', () => {
|
||||
expect(error).toBeInstanceOf(TerminalSessionOwnerUnverifiedError)
|
||||
expect(error.name).toBe('TerminalSessionOwnerUnverifiedError')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { PtyWriteUnavailableError } from '../providers/pty-write-unavailable-error'
|
||||
// Error classes shared across the daemon protocol boundary (client, server,
|
||||
// host). Split from types.ts, which is capped for wire-shape declarations.
|
||||
export class TerminalAttachCanceledError extends Error {
|
||||
@@ -22,15 +21,7 @@ export class SessionNotFoundError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A PtyWriteUnavailableError so a throw partway through a paste reaches the renderer as
|
||||
* `pty:writeUnavailable` and the pane re-attaches, instead of the remaining chunks vanishing
|
||||
* with nothing to explain the gap.
|
||||
*
|
||||
* Deliberately not a SessionNotFoundError: that is matched by isPtyAlreadyGoneError and would
|
||||
* be synthesized into an exit the session never had — the same lie one layer down.
|
||||
*/
|
||||
export class TerminalSessionOwnerUnverifiedError extends PtyWriteUnavailableError {
|
||||
export class TerminalSessionOwnerUnverifiedError extends Error {
|
||||
constructor(sessionId: string) {
|
||||
super(`Terminal session owner could not be verified: ${sessionId}`)
|
||||
this.name = 'TerminalSessionOwnerUnverifiedError'
|
||||
|
||||
@@ -23,8 +23,6 @@ import {
|
||||
} from './daemon-health'
|
||||
import type { SubprocessHandle } from './session'
|
||||
|
||||
// Why: the veto's production default is otherwise never exercised — every other test injects it.
|
||||
|
||||
function createMockSubprocess(): SubprocessHandle {
|
||||
return {
|
||||
pid: 55555,
|
||||
|
||||
@@ -28,9 +28,8 @@ import {
|
||||
type SystemResolverHealthResult
|
||||
} from './types'
|
||||
|
||||
export const HEALTH_CHECK_TIMEOUT_MS = 3_000
|
||||
/** Ceiling on one identity `ps`; the launch budget reserves against it, so it is exported rather than inline. */
|
||||
export const PS_IDENTITY_TIMEOUT_MS = 2_000
|
||||
const HEALTH_CHECK_TIMEOUT_MS = 3_000
|
||||
const PS_IDENTITY_TIMEOUT_MS = 2_000
|
||||
const RESOLVER_HEALTH_CHECK_TIMEOUT_MS = 3_000
|
||||
const KILL_WAIT_MS = 3_000
|
||||
const KILL_POLL_MS = 100
|
||||
@@ -453,7 +452,7 @@ function getPsProcessIdentity(pid: number): PsProcessIdentity | null {
|
||||
try {
|
||||
const output = execFileSync('ps', ['-p', String(pid), '-o', 'lstart=', '-o', 'command='], {
|
||||
encoding: 'utf8',
|
||||
timeout: PS_IDENTITY_TIMEOUT_MS
|
||||
timeout: 2_000
|
||||
})
|
||||
return parsePsProcessIdentity(output)
|
||||
} catch {
|
||||
@@ -642,7 +641,7 @@ export async function getDaemonLaunchIdentity(
|
||||
return commandLine.includes(expectedEntryPath) ? 'match' : 'mismatch'
|
||||
}
|
||||
|
||||
export async function readVerifiedDaemonPid(
|
||||
async function readVerifiedDaemonPid(
|
||||
runtimeDir: string,
|
||||
socketPath: string,
|
||||
tokenPath: string,
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
/* eslint-disable max-lines -- Why: covers daemon-init's full restart flow (7-step sequence per docs/daemon-staleness-ux.md §Phase 1 + coalescer); one describe block keeps shared mocks in one place. */
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS } from './daemon-init'
|
||||
import { OCCUPANCY_CONNECT_BUDGET_MS, OCCUPANCY_REQUEST_BUDGET_MS } from './daemon-occupancy'
|
||||
import type { DaemonLaunchMode } from './daemon-spawner'
|
||||
import { join } from 'node:path'
|
||||
import { PROTOCOL_VERSION } from './types'
|
||||
import { WEDGED_DAEMON_GRACE_RETRIES } from './daemon-init'
|
||||
|
||||
const FAKE_USER_DATA_PATH = '/fake/userData'
|
||||
const FAKE_RUNTIME_DIR = join(FAKE_USER_DATA_PATH, 'daemon')
|
||||
@@ -30,8 +28,6 @@ const {
|
||||
getDaemonLaunchIdentityMock,
|
||||
isDaemonStaleForCurrentBundleMock,
|
||||
killStaleDaemonMock,
|
||||
readVerifiedDaemonPidMock,
|
||||
inspectDaemonPtyOwnershipMock,
|
||||
getProcessStartedAtMsMock,
|
||||
parseDaemonPidFileMock,
|
||||
replaceDaemonPidFileMock,
|
||||
@@ -95,10 +91,6 @@ const {
|
||||
const getMacDaemonTccAttributionHealthMock = vi.fn(async () => 'unknown')
|
||||
const getDaemonLaunchIdentityMock = vi.fn(() => 'match')
|
||||
const isDaemonStaleForCurrentBundleMock = vi.fn(() => false)
|
||||
const inspectDaemonPtyOwnershipMock = vi.fn(
|
||||
async (): Promise<'owns-live-ptys' | 'no-live-ptys' | 'unknown'> => 'unknown'
|
||||
)
|
||||
const readVerifiedDaemonPidMock = vi.fn(async (): Promise<{ pid: number } | null> => null)
|
||||
const killStaleDaemonMock = vi.fn(async () => ({
|
||||
killed: true,
|
||||
liveOwnerSurvived: false
|
||||
@@ -137,12 +129,10 @@ const {
|
||||
|
||||
// Why: every DaemonSpawner pushes here so assertions can check the *same* spawner was reused across restart.
|
||||
const spawnerInstances: MockSpawner[] = []
|
||||
// Mirrors DaemonLaunchMode rather than restating one of its members: the type used to omit
|
||||
// 'held', so no test could describe the launch the hold produces.
|
||||
const ensureRunningOverrides: (() => Promise<{
|
||||
socketPath: string
|
||||
tokenPath: string
|
||||
mode?: DaemonLaunchMode
|
||||
mode?: 'degraded-new-pty-fallback'
|
||||
}>)[] = []
|
||||
const adoptionLeaseReleases: ReturnType<typeof vi.fn>[] = []
|
||||
const lifecycleLeaseErrors: Error[] = []
|
||||
@@ -206,8 +196,6 @@ const {
|
||||
getDaemonLaunchIdentityMock,
|
||||
isDaemonStaleForCurrentBundleMock,
|
||||
killStaleDaemonMock,
|
||||
readVerifiedDaemonPidMock,
|
||||
inspectDaemonPtyOwnershipMock,
|
||||
getProcessStartedAtMsMock,
|
||||
parseDaemonPidFileMock,
|
||||
replaceDaemonPidFileMock,
|
||||
@@ -277,17 +265,6 @@ vi.mock('electron', () => ({
|
||||
}
|
||||
}))
|
||||
|
||||
// Map the existing boolean socket double onto the canonical three-valued probe:
|
||||
// present ⇒ something is serving, absent ⇒ positively dead.
|
||||
vi.mock('./daemon-live-pty-evidence', () => ({
|
||||
inspectDaemonPtyOwnership: inspectDaemonPtyOwnershipMock
|
||||
}))
|
||||
|
||||
vi.mock('./daemon-endpoint-probe', async (importOriginal) => ({
|
||||
...(await importOriginal<Record<string, unknown>>()),
|
||||
probeSocketConnect: async (p: string) => (probeSocketExistsMock(p) ? 'connected' : 'missing')
|
||||
}))
|
||||
|
||||
vi.mock('fs', () => ({
|
||||
mkdirSync: vi.fn(),
|
||||
existsSync: (p: string) => probeSocketExistsMock(p) || p.includes('.pid'),
|
||||
@@ -312,26 +289,11 @@ vi.mock('./daemon-health', () => ({
|
||||
healthCheckDaemon: healthCheckDaemonMock,
|
||||
isDaemonStaleForCurrentBundle: isDaemonStaleForCurrentBundleMock,
|
||||
killStaleDaemon: killStaleDaemonMock,
|
||||
readVerifiedDaemonPid: readVerifiedDaemonPidMock,
|
||||
getProcessStartedAtMs: getProcessStartedAtMsMock,
|
||||
parseDaemonPidFile: parseDaemonPidFileMock
|
||||
}))
|
||||
|
||||
vi.mock('./client', () => ({
|
||||
// Mirror ensureConnected onto the bounded variant so every existing double keeps its
|
||||
// behaviour — including the ones whose whole point is that connecting throws. The budget is
|
||||
// forwarded rather than dropped so a test can see what the launcher was willing to wait.
|
||||
DaemonClient: function DaemonClientDouble(...args: unknown[]) {
|
||||
const instance = (daemonClientMock as unknown as (...a: unknown[]) => Record<string, unknown>)(
|
||||
...args
|
||||
)
|
||||
if (instance && typeof instance === 'object' && !('ensureConnectedWithin' in instance)) {
|
||||
instance.ensureConnectedWithin = (budgetMs?: number) =>
|
||||
(instance.ensureConnected as (ms?: number) => unknown)(budgetMs)
|
||||
}
|
||||
return instance
|
||||
}
|
||||
}))
|
||||
vi.mock('./client', () => ({ DaemonClient: daemonClientMock }))
|
||||
|
||||
vi.mock('./daemon-lifecycle-event', () => ({
|
||||
trackDaemonReplaced: trackDaemonReplacedMock,
|
||||
@@ -347,7 +309,7 @@ vi.mock('./daemon-spawner', () => ({
|
||||
readonly getHandle: ReturnType<typeof vi.fn>
|
||||
private socketCounter: number
|
||||
private handle: {
|
||||
mode?: DaemonLaunchMode
|
||||
mode?: 'degraded-new-pty-fallback'
|
||||
releaseAdoptionLease?: () => void
|
||||
shutdown: () => Promise<void>
|
||||
} | null
|
||||
@@ -592,10 +554,6 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks()
|
||||
// Why restore too: tests here spy on Date.now, and clearAllMocks keeps the fake
|
||||
// implementation. A failure before an inline restore would freeze the clock for every
|
||||
// later test in the file, turning one red into a cascade.
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('re-binds listeners after the first daemon provider is installed', async () => {
|
||||
@@ -759,28 +717,6 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
expect(adapterInstances[1].disconnectOnly).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('routes a held daemon through the degraded provider, not a bare adapter', async () => {
|
||||
// The wiring the hold depends on. Without it a held daemon gets a plain DaemonPtyAdapter:
|
||||
// every fresh spawn hangs on the wedged socket instead of falling back locally, and
|
||||
// isDaemonDegraded() — an instanceof check — reports false, so the notice telling the user
|
||||
// how to recover never renders.
|
||||
const mod = await importFresh()
|
||||
ensureRunningOverrides.push(async () => ({
|
||||
socketPath: '/fake/held-socket',
|
||||
tokenPath: '/fake/held-token',
|
||||
mode: 'held'
|
||||
}))
|
||||
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const { DegradedDaemonPtyProvider } = await import('./degraded-daemon-pty-provider')
|
||||
const provider = mod.getDaemonProvider()
|
||||
expect(provider).toBeInstanceOf(DegradedDaemonPtyProvider)
|
||||
expect(
|
||||
(provider as InstanceType<typeof DegradedDaemonPtyProvider>).routesFreshSpawnsToLocalProvider
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('routes fresh PTYs to the local fallback when a preserved daemon cannot spawn new PTYs', async () => {
|
||||
const mod = await importFresh()
|
||||
ensureRunningOverrides.push(async () => ({
|
||||
@@ -1260,50 +1196,6 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('falls back to killStaleDaemon without the live-PTY veto, so an explicit restart always wins', async () => {
|
||||
// Why: this is the documented escape hatch (Settings → Manage Sessions → Restart). Opting
|
||||
// into the veto here would report liveOwnerSurvived and throw, leaving the user no daemon.
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(() => {
|
||||
const handlers: Record<string, (() => void)[]> = { connect: [], error: [] }
|
||||
return {
|
||||
on(event: string, cb: () => void) {
|
||||
handlers[event]?.push(cb)
|
||||
if (event === 'connect') {
|
||||
queueMicrotask(() => cb())
|
||||
}
|
||||
return this
|
||||
},
|
||||
removeListener(event: string, cb: () => void) {
|
||||
handlers[event] = handlers[event]?.filter((handler) => handler !== cb) ?? []
|
||||
return this
|
||||
},
|
||||
destroy() {}
|
||||
}
|
||||
})
|
||||
const mod = await importFresh()
|
||||
daemonClientMock.mockImplementationOnce(function MockWedgedDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
throw new Error('Hello response timed out')
|
||||
}),
|
||||
request: vi.fn(),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
mod.cleanupDaemonForProtocol('/fake/daemon', PROTOCOL_VERSION)
|
||||
).resolves.toMatchObject({ cleaned: true })
|
||||
|
||||
expect(killStaleDaemonMock).toHaveBeenCalledWith(
|
||||
'/fake/daemon',
|
||||
`/fake/daemon/daemon-v${PROTOCOL_VERSION}.sock`,
|
||||
`/fake/daemon/daemon-v${PROTOCOL_VERSION}.token`,
|
||||
PROTOCOL_VERSION
|
||||
)
|
||||
})
|
||||
|
||||
it('coalesces concurrent restartDaemon() calls so the 7-step sequence runs exactly once', async () => {
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
@@ -1837,7 +1729,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
'/fake/token',
|
||||
FAKE_DAEMON_ENTRY_PATH
|
||||
)
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(disconnectMock).toHaveBeenCalledOnce()
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
@@ -1871,7 +1763,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(disconnectMock).toHaveBeenCalledOnce()
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
@@ -1970,7 +1862,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(getMacDaemonSystemResolverHealthMock).toHaveBeenCalledWith('/fake/socket', '/fake/token')
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(disconnectMock).toHaveBeenCalledOnce()
|
||||
expect(getDaemonLaunchIdentityMock).not.toHaveBeenCalled()
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
@@ -2007,7 +1899,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(disconnectMock).toHaveBeenCalledOnce()
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
@@ -2948,7 +2840,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(disconnectMock).toHaveBeenCalledOnce()
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
@@ -2979,23 +2871,20 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{
|
||||
mode?: DaemonLaunchMode
|
||||
mode?: 'degraded-new-pty-fallback'
|
||||
shutdown(): Promise<void>
|
||||
}>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('pty-spawn-unhealthy')
|
||||
|
||||
const handle = await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(handle.mode).toBe('degraded-new-pty-fallback')
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Note: only the adoption client is wedged here; the session probe answers, so this is the
|
||||
// verified-zero case and the live-PTY veto stays off. The unverifiable path is covered by
|
||||
// the grace-retry tests below.
|
||||
it('replaces a health-check-failing daemon that reports no sessions when the pipe is dead', async () => {
|
||||
it('replaces a health-check-failing daemon when live sessions cannot be verified and the pipe is dead', async () => {
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
@@ -3062,8 +2951,6 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
daemonClientMock
|
||||
.mockImplementationOnce(unreachableClient)
|
||||
.mockImplementationOnce(unreachableClient)
|
||||
// A cold start has no pid record to act on, so nothing is ever killed.
|
||||
killStaleDaemonMock.mockResolvedValueOnce({ killed: false, liveOwnerSurvived: false })
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
@@ -3160,12 +3047,8 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('holds a permanently wedged daemon rather than killing what it might be hosting', async () => {
|
||||
// The trade this makes, deliberately: a socket that accepts connections but never answers
|
||||
// hello can no longer be replaced at launch, so a wedged-but-empty daemon stays until the
|
||||
// user restarts it (#8689 regresses to degraded mode). The alternative was killing a daemon
|
||||
// whose live agents we had simply failed to observe, and that loss is unrecoverable while
|
||||
// this one is one click from repaired.
|
||||
it('replaces a permanently wedged daemon after the grace window is exhausted (#8689)', async () => {
|
||||
// Why: a socket that accepts connections but never answers hello was preserved forever (#8689); after grace it must be replaced.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
@@ -3182,7 +3065,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
daemonClientConstructionCount++
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
if (daemonClientConstructionCount <= 2) {
|
||||
if (daemonClientConstructionCount <= 2 + WEDGED_DAEMON_GRACE_RETRIES) {
|
||||
throw new Error('Hello response timed out')
|
||||
}
|
||||
}),
|
||||
@@ -3221,22 +3104,22 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
try {
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
// 1 adoption + 1 patient probe. No post-fork adoption, because nothing was forked, and
|
||||
// no retries, because there is no longer a retry loop: what remains after the patient
|
||||
// connect is always exactly OCCUPANCY_REQUEST_BUDGET_MS, which cannot fund another ask
|
||||
// at any ceiling.
|
||||
expect(daemonClientMock).toHaveBeenCalledTimes(2)
|
||||
// The verdict must still be recorded: holding without a log is indistinguishable from
|
||||
// a successful adoption to anyone reading the log afterwards.
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('holding an unreachable daemon'))
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Manage Sessions'))
|
||||
// Why the second remedy is asserted: killStaleDaemon only kills a process whose identity
|
||||
// matches the pid record, so when something other than an Orca daemon holds the endpoint
|
||||
// a Restart clears nothing and the next launch is identical. Offering only the remedy
|
||||
// that cannot work is how a user concludes the app is broken.
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('quit and relaunch'))
|
||||
expect(killStaleDaemonMock).toHaveBeenCalledWith(
|
||||
FAKE_RUNTIME_DIR,
|
||||
'/fake/socket',
|
||||
'/fake/token'
|
||||
)
|
||||
expect(forkMock).toHaveBeenCalled()
|
||||
// The launcher probes the full grace budget: 1 initial probe + WEDGED_DAEMON_GRACE_RETRIES retries.
|
||||
expect(daemonClientMock).toHaveBeenCalledTimes(3 + WEDGED_DAEMON_GRACE_RETRIES)
|
||||
// Why: this replace path used to kill the daemon with no log, so a post-hoc
|
||||
// reader could not tell it apart from an adoption; the verdict must be recorded.
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Replacing daemon that failed the health check')
|
||||
)
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining(`graceRetries=${WEDGED_DAEMON_GRACE_RETRIES}`)
|
||||
)
|
||||
} finally {
|
||||
warnSpy.mockRestore()
|
||||
// Restore the answering default: clearAllMocks clears calls not impls, so the throwing impl would leak into later tests.
|
||||
@@ -3244,373 +3127,13 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('holds a wedged daemon that still owns live terminals instead of killing its agents', async () => {
|
||||
// Why hold and not adopt: a daemon that cannot answer listSessions cannot answer a hello,
|
||||
// so there is nothing to adopt. The kill must never be attempted — its agents are alive.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const answeringDefault = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
daemonClientMock.mockImplementation(function MockWedgedDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
throw new Error('Hello response timed out')
|
||||
}),
|
||||
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
|
||||
request: vi.fn(),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
// The daemon is identity-verified and its process still owns live terminals.
|
||||
readVerifiedDaemonPidMock.mockResolvedValue({ pid: 4242 })
|
||||
inspectDaemonPtyOwnershipMock.mockResolvedValue('owns-live-ptys')
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void>; mode?: string }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('unreachable')
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
const handle = await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(handle.mode).toBe('held')
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
readVerifiedDaemonPidMock.mockResolvedValue(null)
|
||||
inspectDaemonPtyOwnershipMock.mockResolvedValue('unknown')
|
||||
}
|
||||
it('grace budget is generous enough to ride out a ~60s transient wedge', () => {
|
||||
// Why: each probe waits the client's 5s hello timeout, so 1 + 11 probes ≈ 60s of drain grace; don't cut without telemetry.
|
||||
expect(WEDGED_DAEMON_GRACE_RETRIES).toBeGreaterThanOrEqual(11)
|
||||
})
|
||||
|
||||
it('adopts a PTY-spawn-unhealthy daemon in degraded mode when it can still answer', async () => {
|
||||
// Why the count must come from IPC: an answered listSessions is what proves the daemon
|
||||
// can still complete a handshake, and adoption opens one.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const answeringDefault = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
daemonClientMock.mockImplementation(function MockAnsweringDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [{ isAlive: true }] })),
|
||||
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void>; mode?: string }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('pty-spawn-unhealthy')
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
const handle = await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(handle.mode).toBe('degraded-new-pty-fallback')
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
}
|
||||
})
|
||||
|
||||
it('holds a PTY-spawn-unhealthy daemon that has since stopped answering', async () => {
|
||||
// Why: `health` is a reading from before the grace window. If listSessions went
|
||||
// unanswered across all of it, adoption would open a hello the daemon can no longer
|
||||
// complete — and that throw costs the app its daemon.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const answeringDefault = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
daemonClientMock.mockImplementation(function MockSilentDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => {
|
||||
throw new Error('listSessions timed out')
|
||||
}),
|
||||
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
readVerifiedDaemonPidMock.mockResolvedValue({ pid: 4242 })
|
||||
inspectDaemonPtyOwnershipMock.mockResolvedValue('owns-live-ptys')
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void>; mode?: string }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('pty-spawn-unhealthy')
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
const handle = await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(handle.mode).toBe('held')
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
readVerifiedDaemonPidMock.mockResolvedValue(null)
|
||||
inspectDaemonPtyOwnershipMock.mockResolvedValue('unknown')
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a daemon handle when the preserved daemon is too wedged to be adopted', async () => {
|
||||
// Why: adoption needs a hello, which is exactly what a daemon wedged enough to be
|
||||
// preserved cannot answer. Throwing here would abort initDaemonPtyProvider, leaving no
|
||||
// spawner — and restartDaemon() throws without one, so the user loses the documented
|
||||
// Manage Sessions → Restart remedy on top of having no daemon.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const answeringDefault = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
// Permanently wedged: every client, including the adoption client, fails its hello.
|
||||
daemonClientMock.mockImplementation(function MockWedgedDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
throw new Error('Hello response timed out')
|
||||
}),
|
||||
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
|
||||
request: vi.fn(),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
killStaleDaemonMock.mockResolvedValueOnce({ killed: false, liveOwnerSurvived: true })
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void>; mode?: string }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('unreachable')
|
||||
// The endpoint still listens, so the daemon is wedged rather than gone.
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
const handle = await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
// 'held', not merely degraded: init must not attempt a lease on a daemon whose
|
||||
// adoption hello just failed — that throw would cost the app its daemon entirely.
|
||||
expect(handle.mode).toBe('held')
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
}
|
||||
})
|
||||
|
||||
it('holds a hello-rejected daemon that owns live terminals rather than adopting it', async () => {
|
||||
// Why: 'rejected' means it answered and refused the handshake, so adoption can never
|
||||
// succeed. Falling through to preserveDaemon() would throw and cost the app its daemon
|
||||
// entirely — and killing it would end agents that are still running.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const answeringDefault = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
daemonClientMock.mockImplementation(function MockRejectingDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
throw new Error('hello refused')
|
||||
}),
|
||||
getDaemonIdentity: vi.fn(readLaunchedDaemonIdentity),
|
||||
request: vi.fn(),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
readVerifiedDaemonPidMock.mockResolvedValue({ pid: 4242 })
|
||||
inspectDaemonPtyOwnershipMock.mockResolvedValue('owns-live-ptys')
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void>; mode?: string }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('rejected')
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
const handle = await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(handle.mode).toBe('held')
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
readVerifiedDaemonPidMock.mockResolvedValue(null)
|
||||
inspectDaemonPtyOwnershipMock.mockResolvedValue('unknown')
|
||||
}
|
||||
})
|
||||
|
||||
const wedgedClient = function MockWedgedDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
throw new Error('Hello response timed out')
|
||||
}),
|
||||
request: vi.fn(),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
const answeringClient = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
it('still replaces when the endpoint is proven dead, so a cold start is not held', async () => {
|
||||
// The regression holding most risks: 'unknown' is also what a cold start looks like, since
|
||||
// nothing answers when nothing is there. Holding then would hand every first launch a
|
||||
// provider pointed at no daemon. A missing socket is the discriminator.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
daemonClientMock.mockImplementation(wedgedClient)
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void> }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('unreachable')
|
||||
probeSocketExistsMock.mockReturnValue(false)
|
||||
// Reaching the fork IS the assertion; throwing there stops before the spawn plumbing.
|
||||
forkMock.mockImplementationOnce(() => {
|
||||
throw new Error('reached the replacement fork')
|
||||
})
|
||||
|
||||
try {
|
||||
await expect(launcher('/fake/socket', '/fake/token')).rejects.toThrow(
|
||||
'reached the replacement fork'
|
||||
)
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringClient)
|
||||
}
|
||||
})
|
||||
|
||||
it('still replaces an unreachable daemon that refused the handshake', async () => {
|
||||
// 'rejected' answered and refused — bad token or foreign protocol — so it can never be
|
||||
// adopted and its sessions can never be reattached. Holding one would be permanent
|
||||
// degradation buying nothing, which is the opposite of the trade holding exists to make.
|
||||
// Note this needs occupancy to stay 'unknown': a daemon that answers listSessions is
|
||||
// 'empty' and reaches the replace path without ever consulting the exclusion.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
daemonClientMock.mockImplementation(wedgedClient)
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void> }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('rejected')
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
forkMock.mockImplementationOnce(() => {
|
||||
throw new Error('reached the replacement fork')
|
||||
})
|
||||
|
||||
try {
|
||||
await expect(launcher('/fake/socket', '/fake/token')).rejects.toThrow(
|
||||
'reached the replacement fork'
|
||||
)
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringClient)
|
||||
}
|
||||
})
|
||||
|
||||
it('spends a patient connect budget on the wedged ask, not the cheap one', async () => {
|
||||
// Pins the round-11 defect, which shipped green because every other test recomputes the
|
||||
// budget expression instead of watching the launcher spend it: with the evidence clock
|
||||
// withheld, max(CONNECT, probeBudget - REQUEST) collapsed to CONNECT and the "patient" ask
|
||||
// was a cheap ask wearing a comment. Observed here through the client double, which
|
||||
// forwards the budget rather than dropping it.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
const connectBudgets: (number | undefined)[] = []
|
||||
const answeringDefault = function MockDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {}),
|
||||
request: vi.fn(async () => ({ sessions: [] })),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
}
|
||||
daemonClientMock.mockImplementation(function MockWedgedDaemonClient() {
|
||||
return {
|
||||
ensureConnected: vi.fn(async (budgetMs?: number) => {
|
||||
connectBudgets.push(budgetMs)
|
||||
throw new Error('Hello response timed out')
|
||||
}),
|
||||
request: vi.fn(),
|
||||
disconnect: vi.fn()
|
||||
}
|
||||
})
|
||||
|
||||
const launcher = spawnerInstances[0].launcher as (
|
||||
socketPath: string,
|
||||
tokenPath: string
|
||||
) => Promise<{ shutdown(): Promise<void> }>
|
||||
checkDaemonHealthMock.mockResolvedValueOnce('unreachable')
|
||||
probeSocketExistsMock.mockReturnValue(true)
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
// The first budget is the launcher's own adoption connect; the ask follows it.
|
||||
const askBudget = connectBudgets[1]
|
||||
expect(askBudget).toBeGreaterThan(OCCUPANCY_CONNECT_BUDGET_MS)
|
||||
// And it must still leave the answer something to arrive in.
|
||||
expect(askBudget).toBeLessThanOrEqual(
|
||||
WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS - OCCUPANCY_REQUEST_BUDGET_MS
|
||||
)
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
}
|
||||
})
|
||||
|
||||
it('adopts a daemon that drains inside the patient ask, rather than degrading it', async () => {
|
||||
// The payoff for spending the clock on one tolerant ask instead of many cheap ones. A
|
||||
// counted answer is the only verdict that reaches preserveDaemon() — full daemon service —
|
||||
// where 'unknown' would have settled for a degraded hold. This is what a slow-but-alive
|
||||
// daemon gets back.
|
||||
const frozenNow = Date.now()
|
||||
vi.spyOn(Date, 'now').mockReturnValue(frozenNow)
|
||||
it('preserves a daemon that stays wedged until the LAST allowed grace retry', async () => {
|
||||
// Why: daemon drains only on the last allowed probe (1 + WEDGED_DAEMON_GRACE_RETRIES) — must be preserved, not replaced.
|
||||
const mod = await importFresh()
|
||||
await mod.initDaemonPtyProvider()
|
||||
|
||||
@@ -3624,8 +3147,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
}
|
||||
daemonClientMock.mockImplementation(function MockDaemonClient() {
|
||||
probe += 1
|
||||
// 1 = the launcher's adoption client; 2 = the patient ask, which is where it drains.
|
||||
const drainsNow = probe >= 2
|
||||
const drainsNow = probe >= 1 + WEDGED_DAEMON_GRACE_RETRIES
|
||||
return {
|
||||
ensureConnected: vi.fn(async () => {
|
||||
if (!drainsNow) {
|
||||
@@ -3648,13 +3170,10 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
netConnectMock.mockImplementation(stubAliveSocketConnect)
|
||||
|
||||
try {
|
||||
const handle = (await launcher('/fake/socket', '/fake/token')) as { mode?: string }
|
||||
await launcher('/fake/socket', '/fake/token')
|
||||
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
// Not 'held': a counted answer is adoptable, and settling for degraded mode here would
|
||||
// waste the very patience the single ask was widened to buy.
|
||||
expect(handle.mode).toBeUndefined()
|
||||
} finally {
|
||||
daemonClientMock.mockImplementation(answeringDefault)
|
||||
}
|
||||
@@ -3984,7 +3503,7 @@ describe('daemon-init: runRestartDaemon (7-step sequence)', () => {
|
||||
'/fake/token',
|
||||
'1.2.3'
|
||||
)
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined, expect.any(Number))
|
||||
expect(requestMock).toHaveBeenCalledWith('listSessions', undefined)
|
||||
expect(disconnectMock).toHaveBeenCalledOnce()
|
||||
expect(killStaleDaemonMock).not.toHaveBeenCalled()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
|
||||
+68
-245
@@ -15,18 +15,9 @@ import {
|
||||
unlinkOwnedDaemonPidFile,
|
||||
type DaemonLauncher,
|
||||
type DaemonPidFile,
|
||||
type DaemonLaunchMode,
|
||||
type DaemonProcessHandle
|
||||
} from './daemon-spawner'
|
||||
import { DAEMON_EXIT_ENDPOINT_OCCUPIED } from './daemon-endpoint-ownership'
|
||||
import { endpointIsProvenDead, probeSocketConnect } from './daemon-endpoint-probe'
|
||||
import {
|
||||
OCCUPANCY_CONNECT_BUDGET_MS,
|
||||
OCCUPANCY_REQUEST_BUDGET_MS,
|
||||
raiseOccupancyWithProcessEvidence,
|
||||
resolveDaemonOccupancy,
|
||||
type DaemonOccupancy
|
||||
} from './daemon-occupancy'
|
||||
import { DaemonPtyAdapter, type DaemonRespawnReason } from './daemon-pty-adapter'
|
||||
import { DaemonPtyRouter } from './daemon-pty-router'
|
||||
import { DaemonClient } from './client'
|
||||
@@ -43,7 +34,6 @@ import {
|
||||
checkDaemonHealth,
|
||||
isDaemonStaleForCurrentBundle,
|
||||
killStaleDaemon,
|
||||
readVerifiedDaemonPid,
|
||||
parseDaemonPidFile,
|
||||
type MacDaemonTccAttributionHealth
|
||||
} from './daemon-health'
|
||||
@@ -80,47 +70,8 @@ function logDaemonMilestone(event: string, details: Record<string, unknown> = {}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ceiling on the whole failed-health classification — every probe, the grace window, the
|
||||
* identity check and the process-table read together — enforced at runtime rather than
|
||||
* summed by hand.
|
||||
*
|
||||
* Why enforced: startup fails open at 60s by abandoning the daemon provider outright, and
|
||||
* ensureRunning() is not abortable, so overrunning costs the app its daemon *and* still kills
|
||||
* the incumbent. Four separate reviews found a term missing from the hand-written sum that
|
||||
* was supposed to prevent that — the launcher's own adoption connect, an identity probe, an
|
||||
* endpoint probe, a doubled evidence deadline. A budget that has to be remembered is a budget
|
||||
* that will be wrong, so the code now spends against a clock and stops when it runs out.
|
||||
*
|
||||
* The remainder of the fail-open window belongs to what follows a replace verdict: the kill
|
||||
* ladder and the daemon fork.
|
||||
*/
|
||||
export const WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS = 34_000
|
||||
|
||||
/**
|
||||
* The clock the identity re-check and process-table read need, checked *after* the probes
|
||||
* rather than withheld from them.
|
||||
*
|
||||
* It used to be a reservation, and that was backwards: withholding twelve seconds starved the
|
||||
* one probe whose answer can still restore full daemon mode, since only a counted reply reaches
|
||||
* preserveDaemon(). With the reservation in place the "patient" ask resolved to exactly the
|
||||
* cheap ask's four seconds.
|
||||
*
|
||||
* Do not read the demotion as "the evidence read is cosmetic" — an earlier version of this
|
||||
* comment said that and it was wrong twice over. The read decides the verdict wherever the
|
||||
* unknown hold declines to: it has no endpointIsProvenDead check and no health !== 'rejected'
|
||||
* check, so evidence is what holds a daemon whose socket entry vanished, and what holds a
|
||||
* hello-rejected daemon that is still hosting agents. Skipping it on a spent clock therefore
|
||||
* withdraws real protection, not a log line; the gate is set so that only probes which already
|
||||
* consumed the budget can trigger it.
|
||||
*
|
||||
* So the evidence read is opportunistic now. If the probes used the clock, it is skipped and
|
||||
* the verdict stays 'unknown' — which holds the daemon exactly as an evidence-raised
|
||||
* 'occupied' would have. Nothing is lost but a more precise log line.
|
||||
*
|
||||
* Zero on Windows, which runs neither step.
|
||||
*/
|
||||
export const CLASSIFICATION_EVIDENCE_MIN_MS = process.platform === 'win32' ? 0 : 12_000
|
||||
// Why: extra hello+listSessions probes (~5s each) giving a wedged-but-connectable daemon ~60s grace to answer and keep its live sessions before a permanent wedge (#8689) is replaced; raise only alongside the fail-open cap.
|
||||
export const WEDGED_DAEMON_GRACE_RETRIES = 11
|
||||
const DAEMON_SELF_SHUTDOWN_WAIT_MS = 5_000
|
||||
const DAEMON_CHILD_TERMINATION_GRACE_MS = 5_000
|
||||
const DAEMON_CHILD_FORCE_EXIT_WAIT_MS = 1_000
|
||||
@@ -222,10 +173,27 @@ function probeSocket(socketPath: string): Promise<boolean> {
|
||||
})
|
||||
}
|
||||
|
||||
async function getAliveDaemonSessionCount(
|
||||
socketPath: string,
|
||||
tokenPath: string,
|
||||
protocolVersion = PROTOCOL_VERSION
|
||||
): Promise<number | null> {
|
||||
const client = new DaemonClient({ socketPath, tokenPath, protocolVersion })
|
||||
try {
|
||||
await client.ensureConnected()
|
||||
const result = await client.request<ListSessionsResult>('listSessions', undefined)
|
||||
return result.sessions.filter((session) => session.isAlive).length
|
||||
} catch {
|
||||
return null
|
||||
} finally {
|
||||
client.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
function createPreservedDaemonHandle(
|
||||
runtimeDir: string,
|
||||
protocolVersion = PROTOCOL_VERSION,
|
||||
mode?: DaemonLaunchMode
|
||||
mode?: 'degraded-new-pty-fallback'
|
||||
): DaemonProcessHandle {
|
||||
const handle: DaemonProcessHandle = {
|
||||
shutdown: async () => {
|
||||
@@ -452,30 +420,19 @@ function isNoSuchProcessError(error: unknown): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ESRCH'
|
||||
}
|
||||
|
||||
/** How a preserve decision reads in a log line, from either evidence source. */
|
||||
function describeOccupancy(occupancy: DaemonOccupancy): string {
|
||||
if (occupancy.liveSessions === null) {
|
||||
return 'live session state could not be verified'
|
||||
}
|
||||
return `it owns ${occupancy.liveSessions} live session${occupancy.liveSessions === 1 ? '' : 's'}`
|
||||
}
|
||||
|
||||
/** IPC only: these callers run against a daemon that just answered a health check. */
|
||||
function resolveOccupancyOverIpc(socketPath: string, tokenPath: string): Promise<DaemonOccupancy> {
|
||||
return resolveDaemonOccupancy({ socketPath, tokenPath, recordedPid: null })
|
||||
}
|
||||
|
||||
async function shouldPreserveDaemonWithLiveSessions(
|
||||
socketPath: string,
|
||||
tokenPath: string,
|
||||
replacementLabel: string
|
||||
): Promise<boolean> {
|
||||
const occupancy = await resolveOccupancyOverIpc(socketPath, tokenPath)
|
||||
if (occupancy.state === 'empty') {
|
||||
const liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
|
||||
if (liveSessionCount === 0) {
|
||||
return false
|
||||
}
|
||||
console.warn(
|
||||
`[daemon] Preserving daemon ${replacementLabel} because ${describeOccupancy(occupancy)}`
|
||||
liveSessionCount === null
|
||||
? `[daemon] Preserving daemon ${replacementLabel} because live session state could not be verified`
|
||||
: `[daemon] Preserving daemon ${replacementLabel} because it owns ${liveSessionCount} live session${liveSessionCount === 1 ? '' : 's'}`
|
||||
)
|
||||
return true
|
||||
}
|
||||
@@ -503,9 +460,6 @@ function createOutOfProcessLauncher(
|
||||
| {
|
||||
reason: Parameters<typeof trackDaemonReplaced>[0]
|
||||
liveSessionCount: number | null
|
||||
/** Rendered once the outcome is known; only the health-check branch announces. */
|
||||
verdict?: string
|
||||
announce?: boolean
|
||||
}
|
||||
| undefined
|
||||
let confirmedReplacement = false
|
||||
@@ -513,35 +467,17 @@ function createOutOfProcessLauncher(
|
||||
socketPath,
|
||||
tokenPath
|
||||
})
|
||||
// Why the clock starts before the adoption connect: that connect is on the classification
|
||||
// path and uses the non-shared five-seconds-per-step default, so it was the fourth term to
|
||||
// go missing from the sum this replaces. A ceiling that starts after part of the work is
|
||||
// the same fiction in a new place.
|
||||
const classificationDeadline = Date.now() + WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS
|
||||
const classificationRemainingMs = (): number => Math.max(0, classificationDeadline - Date.now())
|
||||
try {
|
||||
// Why: acquire the full pair before control-only probes so an expired inherited deadline can't fire in the probe-to-adoption gap.
|
||||
// Capped: this only acquires a lease that preserveDaemon() re-establishes anyway, and a
|
||||
// daemon that accepts the socket but never answers hello would otherwise spend the whole
|
||||
// classification clock here, leaving nothing for the probes that protect its sessions.
|
||||
await adoptionClient.ensureConnectedWithin(
|
||||
Math.min(OCCUPANCY_CONNECT_BUDGET_MS, classificationRemainingMs())
|
||||
)
|
||||
await adoptionClient.ensureConnected()
|
||||
await reconcileDaemonPidOwnership(adoptionClient, pidPath)
|
||||
} catch {
|
||||
adoptionClient.disconnect()
|
||||
adoptionClient = null
|
||||
}
|
||||
/**
|
||||
* Keep the incumbent without talking to it. No adoption, so no lease: the lease only
|
||||
* cancels an adoption watchdog, which cannot fire on a daemon that owns sessions.
|
||||
*/
|
||||
const holdIncumbentDaemon = (): DaemonProcessHandle => {
|
||||
adoptionClient?.disconnect()
|
||||
adoptionClient = null
|
||||
return createPreservedDaemonHandle(runtimeDir, PROTOCOL_VERSION, 'held')
|
||||
}
|
||||
const preserveDaemon = async (mode?: DaemonLaunchMode): Promise<DaemonProcessHandle> => {
|
||||
const preserveDaemon = async (
|
||||
mode?: 'degraded-new-pty-fallback'
|
||||
): Promise<DaemonProcessHandle> => {
|
||||
const connectedClient = adoptionClient ?? undefined
|
||||
adoptionClient = null
|
||||
return holdDaemonAdoptionLease(
|
||||
@@ -558,17 +494,19 @@ function createOutOfProcessLauncher(
|
||||
if (health === 'healthy') {
|
||||
const resolverHealth = await getMacDaemonSystemResolverHealth(socketPath, tokenPath)
|
||||
if (resolverHealth === 'unhealthy') {
|
||||
const occupancy = await resolveOccupancyOverIpc(socketPath, tokenPath)
|
||||
if (occupancy.state !== 'empty') {
|
||||
const liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
|
||||
if (liveSessionCount !== 0) {
|
||||
console.warn(
|
||||
`[daemon] Preserving daemon with unavailable macOS system resolver because ${describeOccupancy(occupancy)}`
|
||||
liveSessionCount === null
|
||||
? '[daemon] Preserving daemon with unavailable macOS system resolver because live session state could not be verified'
|
||||
: `[daemon] Preserving daemon with unavailable macOS system resolver because it owns ${liveSessionCount} live session${liveSessionCount === 1 ? '' : 's'}`
|
||||
)
|
||||
return preserveDaemon()
|
||||
}
|
||||
console.warn('[daemon] Replacing daemon with unavailable macOS system resolver')
|
||||
pendingReplacement = {
|
||||
reason: 'unhealthy_resolver',
|
||||
liveSessionCount: 0
|
||||
liveSessionCount
|
||||
}
|
||||
confirmedReplacement = (await cleanupDaemonForProtocol(runtimeDir, PROTOCOL_VERSION))
|
||||
.cleaned
|
||||
@@ -620,12 +558,12 @@ function createOutOfProcessLauncher(
|
||||
if (attributionHealth === 'severed') {
|
||||
// Why: replacing with live sessions would kill them; Settings → Developer
|
||||
// Permissions surfaces the Manage Sessions → Restart remedy instead.
|
||||
const occupancy = await resolveOccupancyOverIpc(socketPath, tokenPath)
|
||||
if (occupancy.state === 'empty') {
|
||||
const liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
|
||||
if (liveSessionCount === 0) {
|
||||
console.warn(
|
||||
'[daemon] Replacing daemon whose macOS TCC attribution is severed (spawning app binary no longer exists)'
|
||||
)
|
||||
pendingReplacement = { reason: 'severed_tcc_attribution', liveSessionCount: 0 }
|
||||
pendingReplacement = { reason: 'severed_tcc_attribution', liveSessionCount }
|
||||
confirmedReplacement = (
|
||||
await cleanupDaemonForProtocol(runtimeDir, PROTOCOL_VERSION)
|
||||
).cleaned
|
||||
@@ -639,141 +577,47 @@ function createOutOfProcessLauncher(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Why: a busy machine can time out the health check on a live daemon; re-verify what
|
||||
// it is hosting before killing its sessions.
|
||||
//
|
||||
// No recordedPid: this loop waits for *IPC* to recover, and the process table cannot
|
||||
// change its answer within a grace window, so scanning it every pass would multiply
|
||||
// the launch budget for an answer we already have. It is read once, after the wait.
|
||||
// Every probe spends the shared clock, minus what the evidence read still needs, so no
|
||||
// probe can eat the reserve however long the daemon takes to answer.
|
||||
const probeBudgetMs = (): number => classificationRemainingMs()
|
||||
const askDaemonWhatItHosts = (connectBudgetMs: number): Promise<DaemonOccupancy> =>
|
||||
resolveDaemonOccupancy({
|
||||
socketPath,
|
||||
tokenPath,
|
||||
recordedPid: null,
|
||||
budgetMs: probeBudgetMs(),
|
||||
connectBudgetMs
|
||||
})
|
||||
// Why the tolerant question goes first: this path is only reached because a 3s health
|
||||
// check timed out, so a cheap probe re-asks on a stricter budget than the one that
|
||||
// triaged the daemon here and can only ever agree with it. The patient ask is the only
|
||||
// one that can disagree, and asking it last meant asking it with the clock already
|
||||
// spent. It gets every millisecond the answer itself does not still need, and never
|
||||
// less than the cheap ask would have had.
|
||||
const patientConnectBudgetMs = (): number =>
|
||||
Math.max(OCCUPANCY_CONNECT_BUDGET_MS, probeBudgetMs() - OCCUPANCY_REQUEST_BUDGET_MS)
|
||||
let occupancy = await askDaemonWhatItHosts(patientConnectBudgetMs())
|
||||
// Why there is no retry loop here any more. Cheap retries used to follow this ask, but
|
||||
// the arithmetic makes them unreachable at every ceiling, not just this one:
|
||||
//
|
||||
// remaining = B - E - max(CONNECT, (B - E) - REQUEST) = REQUEST, whenever B - E > CONNECT + REQUEST
|
||||
//
|
||||
// The patient connect takes every millisecond the answer does not need, so what is left
|
||||
// after it is always exactly OCCUPANCY_REQUEST_BUDGET_MS — never enough to fund another
|
||||
// ask. Raising the budget donates the increase to the same connect and changes nothing.
|
||||
// Funding a real retry needs ~71s of classification, kill ladder and fork against a 60s
|
||||
// fail-open, so the loop cannot be bought back at any price.
|
||||
//
|
||||
// One shape did reach it, and the earlier claim that none could was wrong: a connect
|
||||
// that fails *fast* leaves the budget nearly whole, and while a refused or missing
|
||||
// endpoint is caught by the proven-dead guard, the EPERM/EMFILE class reads 'unknown'
|
||||
// and would have passed. Dropping it costs that case a retry — which retrying was never
|
||||
// going to fix, because an fd-exhausted or permission-denied connect fails the same way
|
||||
// the second time, and recover() puts the daemon back into full service on the next
|
||||
// spawn once the condition clears.
|
||||
//
|
||||
// Nothing is lost by dropping it: a 4s retry cannot reach a daemon that needs longer
|
||||
// than 4s to answer, which is the whole wedge population, while this one ask waits ~12s.
|
||||
// The only case a retry caught and this does not is a daemon that recovers within a few
|
||||
// seconds of being asked — and DegradedDaemonFreshSpawnRouter.recover() already restores
|
||||
// it to full daemon service on the next spawn, off the startup clock entirely.
|
||||
// Do not delete this because 'unknown' and 'occupied' both hold — twice reviewed, twice
|
||||
// proposed for removal, and it regresses both times. The occupied branch below has no
|
||||
// endpointIsProvenDead check and the unknown hold does, so this read is the only thing
|
||||
// standing between a kill and a daemon whose socket entry vanished (a tmp reaper, a
|
||||
// failed publish) while it still hosts live agents: without it that reads as
|
||||
// unknown + proven-dead and falls through to killStaleDaemon.
|
||||
//
|
||||
// The children scan earns its keep for the same reason in reverse: a verified-live pid
|
||||
// alone would also hold a *childless* daemon whose socket vanished, which is the one
|
||||
// #8689 case we can still safely replace.
|
||||
//
|
||||
// The evidence that separates a wedged daemon still hosting terminals from one with
|
||||
// nothing left to lose (#8689). Read once IPC has had its full chance, and only when
|
||||
// it never answered — both because it costs a process scan, and because identity has
|
||||
// to be re-verified first: the grace window is long enough for the daemon to die and
|
||||
// its pid to be recycled, and the evidence would then describe a stranger's children.
|
||||
const evidencePid =
|
||||
occupancy.state === 'unknown' &&
|
||||
process.platform !== 'win32' &&
|
||||
classificationRemainingMs() >= CLASSIFICATION_EVIDENCE_MIN_MS
|
||||
? ((await readVerifiedDaemonPid(runtimeDir, socketPath, tokenPath))?.pid ?? null)
|
||||
: null
|
||||
occupancy = await raiseOccupancyWithProcessEvidence(occupancy, evidencePid)
|
||||
if (occupancy.state === 'occupied') {
|
||||
const owned =
|
||||
occupancy.liveSessions === null
|
||||
? 'live terminal processes'
|
||||
: `${occupancy.liveSessions} live session${occupancy.liveSessions === 1 ? '' : 's'}`
|
||||
|
||||
// Why this comes first: adoption opens a hello, and neither of these daemons can
|
||||
// complete one — 'rejected' answered and refused, and a count only the process table
|
||||
// could supply means nothing answered across the whole grace window. `health` is a
|
||||
// reading from before that window, so it cannot overrule them. Attempting adoption
|
||||
// anyway throws, and the throw costs the app its daemon entirely.
|
||||
if (health === 'rejected' || occupancy.liveSessions === null) {
|
||||
console.warn(
|
||||
`[daemon] DEGRADED MODE: holding a daemon that cannot be adopted (health=${health}) but still owns ${owned}. Killing it would end them; fresh terminals run on the local provider WITHOUT daemon persistence until it recovers or you restart it (Manage Sessions → Restart).`
|
||||
)
|
||||
return holdIncumbentDaemon()
|
||||
}
|
||||
// Why: a busy machine can time out the health check on a live daemon; re-verify with a session list before killing its sessions.
|
||||
let liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
|
||||
// Why: a wedged-but-connectable daemon (Windows update relaunch) may still own live sessions, so grace-retry before replacing; a permanent wedge (#8689) exhausts the grace, and 'rejected' skips it (handshake refused = never adoptable).
|
||||
let graceRetry = 0
|
||||
while (
|
||||
liveSessionCount === null &&
|
||||
health !== 'rejected' &&
|
||||
graceRetry < WEDGED_DAEMON_GRACE_RETRIES &&
|
||||
(await probeSocket(socketPath))
|
||||
) {
|
||||
liveSessionCount = await getAliveDaemonSessionCount(socketPath, tokenPath)
|
||||
graceRetry++
|
||||
}
|
||||
if (liveSessionCount !== null && liveSessionCount > 0) {
|
||||
if (health === 'pty-spawn-unhealthy') {
|
||||
// It answered listSessions just now, so it is adoptable — it simply cannot open
|
||||
// new PTYs.
|
||||
console.warn(
|
||||
`[daemon] DEGRADED MODE: preserving daemon that failed the PTY spawn health check because it owns ${owned}. Existing sessions keep working; fresh terminals run on the local provider WITHOUT daemon persistence until you restart the daemon (Manage Sessions → Restart).`
|
||||
`[daemon] DEGRADED MODE: preserving daemon that failed the PTY spawn health check because it owns ${liveSessionCount} live session${liveSessionCount === 1 ? '' : 's'}. Existing sessions keep working; fresh terminals run on the local provider WITHOUT daemon persistence until you restart the daemon (Manage Sessions → Restart).`
|
||||
)
|
||||
return preserveDaemon('degraded-new-pty-fallback')
|
||||
}
|
||||
console.warn(
|
||||
`[daemon] Preserving daemon that failed the health check because it owns ${owned}`
|
||||
`[daemon] Preserving daemon that failed the health check because it owns ${liveSessionCount} live session${liveSessionCount === 1 ? '' : 's'}`
|
||||
)
|
||||
return preserveDaemon()
|
||||
}
|
||||
// 'unknown' is not permission to kill. Everything above has failed to establish what
|
||||
// this daemon is hosting, and killing it on that basis is what destroyed live agents.
|
||||
// Bounded classification cannot be made safe by budgeting — matching main's tolerance
|
||||
// for one ask costs more clock than the 60s fail-open leaves — so the residual stops
|
||||
// being lethal instead. Being wrong now costs a degraded session, not an agent.
|
||||
//
|
||||
// Two exclusions, both about not holding something that can never be recovered:
|
||||
// - a proven-dead endpoint is a cold start or a corpse; there is nothing to hold, and
|
||||
// holding would hand every first launch a provider pointed at no daemon. Read fresh
|
||||
// here rather than reused, because the daemon can die during the grace window.
|
||||
// - 'rejected' answered and refused the handshake, so it can never be adopted and its
|
||||
// sessions can never be reattached. Holding one is permanently degraded for nothing.
|
||||
if (
|
||||
occupancy.state === 'unknown' &&
|
||||
health !== 'rejected' &&
|
||||
!endpointIsProvenDead(await probeSocketConnect(socketPath))
|
||||
) {
|
||||
// Why: the sibling replace branches announce themselves, but this one used
|
||||
// to kill a daemon silently — leaving no way to tell a replacement apart
|
||||
// from an adoption after the fact. A cold start also lands here with
|
||||
// nothing to replace, so only speak up once something actually answered:
|
||||
// a probe that returned a count, a socket that survived a grace retry, or
|
||||
// a refused hello.
|
||||
if (liveSessionCount !== null || graceRetry > 0 || health === 'rejected') {
|
||||
console.warn(
|
||||
`[daemon] DEGRADED MODE: holding an unreachable daemon (health=${health}); its session state could not be verified, and replacing it would end any terminals it still owns. Fresh terminals run on the local provider WITHOUT daemon persistence until it recovers or you restart it (Manage Sessions → Restart). If a restart does not clear this, something other than an Orca daemon is holding the endpoint — quit and relaunch.`
|
||||
`[daemon] Replacing daemon that failed the health check (health=${health}, liveSessions=${liveSessionCount ?? 'unverifiable'}, graceRetries=${graceRetry})`
|
||||
)
|
||||
return holdIncumbentDaemon()
|
||||
}
|
||||
// Why: a cold start reaches this same fall-through with nothing to replace, so stay
|
||||
// quiet unless something actually answered — a verified count, a socket that survived
|
||||
// a grace retry, or a refused hello.
|
||||
// Why: telemetry gates on confirmedReplacement below — the post-kill truth — so a
|
||||
// cold start that killed nothing never reports a replacement.
|
||||
// Why: unlike the log above, telemetry gates on confirmedReplacement below — the
|
||||
// post-kill truth — so a cold start that killed nothing never reports a replacement.
|
||||
pendingReplacement = {
|
||||
reason: 'failed_health_check',
|
||||
liveSessionCount: occupancy.liveSessions,
|
||||
verdict: `health=${health}, occupancy=${occupancy.state}`,
|
||||
announce: occupancy.state === 'empty' || health === 'rejected'
|
||||
liveSessionCount
|
||||
}
|
||||
}
|
||||
|
||||
@@ -792,25 +636,12 @@ function createOutOfProcessLauncher(
|
||||
try {
|
||||
return await preserveDaemon('degraded-new-pty-fallback')
|
||||
} catch {
|
||||
// Why: adoption needs a hello, which is exactly what a daemon wedged enough to be
|
||||
// preserved cannot answer. A still-listening endpoint means it is wedged, not gone,
|
||||
// so keep a lease-free handle: the lease only cancels the adoption watchdog, which
|
||||
// cannot fire on a daemon that still owns sessions, and throwing here would cost the
|
||||
// app its daemon handle — taking Manage Sessions → Restart down with it.
|
||||
if (!endpointIsProvenDead(await probeSocketConnect(socketPath))) {
|
||||
return createPreservedDaemonHandle(runtimeDir, PROTOCOL_VERSION, 'held')
|
||||
}
|
||||
// It died between the probe and the adoption; the endpoint is genuinely free now.
|
||||
throw new DaemonEndpointOwnershipError(
|
||||
'Daemon replacement aborted: the existing daemon could not be confirmed stopped'
|
||||
)
|
||||
}
|
||||
}
|
||||
if (pendingReplacement?.verdict && (pendingReplacement.announce || killOutcome.killed)) {
|
||||
console.warn(
|
||||
`[daemon] Replacing daemon that failed the health check (${pendingReplacement.verdict})`
|
||||
)
|
||||
}
|
||||
confirmedReplacement = killOutcome.killed || confirmedReplacement
|
||||
// Why: rank by how well each reason is evidenced. A confirmed kill whose reason positively
|
||||
// identified the daemon outranks the attribution, so a stale bundle caught here is not billed
|
||||
@@ -1081,9 +912,6 @@ function createOutOfProcessLauncher(
|
||||
try {
|
||||
return await preserveDaemon('degraded-new-pty-fallback')
|
||||
} catch {
|
||||
// Why not hold here, unlike the failed-health path: that one declined to kill
|
||||
// because it had proof of live work. This one arrives with occupancy unknown or
|
||||
// empty, so holding would swallow a real launch failure to protect nothing.
|
||||
// It stopped answering between the probe and the adoption; report the launch failure.
|
||||
}
|
||||
}
|
||||
@@ -1164,17 +992,12 @@ export async function initDaemonPtyProvider(
|
||||
let routedAdapter: DaemonProvider = newAdapter
|
||||
try {
|
||||
// Why: the launcher's temporary pair closes only after this permanent pair is established, leaving no adoption gap.
|
||||
// Why skipped when held: we deliberately never talked to that daemon, so there is no
|
||||
// handshake to complete and no temporary lease to hand over. Attempting one would throw
|
||||
// and abort init, leaving the app with no spawner — and restartDaemon() throws without one.
|
||||
if (launchMode !== 'held') {
|
||||
await newAdapter.establishLifecycleLease()
|
||||
releaseDaemonAdoptionLease(newSpawner.getHandle())
|
||||
}
|
||||
await newAdapter.establishLifecycleLease()
|
||||
releaseDaemonAdoptionLease(newSpawner.getHandle())
|
||||
|
||||
legacyAdapters = await createLegacyDaemonAdapters(runtimeDir)
|
||||
routedAdapter =
|
||||
launchMode === 'degraded-new-pty-fallback' || launchMode === 'held'
|
||||
launchMode === 'degraded-new-pty-fallback'
|
||||
? new DegradedDaemonPtyProvider({
|
||||
current: newAdapter,
|
||||
legacy: legacyAdapters,
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS } from '../startup/first-window-startup-services'
|
||||
import {
|
||||
CLASSIFICATION_EVIDENCE_MIN_MS,
|
||||
WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS
|
||||
} from './daemon-init'
|
||||
import { OCCUPANCY_CONNECT_BUDGET_MS, OCCUPANCY_REQUEST_BUDGET_MS } from './daemon-occupancy'
|
||||
import {
|
||||
POSIX_OWNERSHIP_PROBE_DEADLINE_MS,
|
||||
PTY_OWNERSHIP_PROBE_ATTEMPTS
|
||||
} from './daemon-live-pty-evidence'
|
||||
import { HEALTH_CHECK_TIMEOUT_MS, PS_IDENTITY_TIMEOUT_MS } from './daemon-health'
|
||||
|
||||
/**
|
||||
* Kept out of the launcher's own spec because that file mocks daemon-health, which would
|
||||
* shadow constants this is here to hold to account.
|
||||
*
|
||||
* This deliberately asserts one enforced ceiling rather than a sum of the path's parts. The
|
||||
* sum was the earlier design, and four separate reviews each found a different term missing
|
||||
* from it — the launcher's own adoption connect, an identity probe, an endpoint probe, an
|
||||
* evidence deadline applied twice. Every one of them passed this file while the real path
|
||||
* overran. The launcher now spends against a clock, so the only thing left worth asserting
|
||||
* is that the clock leaves room for what comes after it.
|
||||
*/
|
||||
describe('wedged-daemon classification budget', () => {
|
||||
it('leaves the kill ladder and the daemon fork room under the startup fail-open', () => {
|
||||
// Startup abandons the daemon provider entirely at the cap, and ensureRunning() is not
|
||||
// abortable — so overrunning costs the app its daemon *and* still kills the incumbent.
|
||||
// What follows a replace verdict is the kill ladder (~11.5s: identity, endpoint probe,
|
||||
// KILL_WAIT, recheck, another probe, SIGKILL confirm) and the fork's own 10s readiness
|
||||
// timeout — plus, on packaged Windows, a daemon-host directory copy of unbounded size.
|
||||
// The margin above 21.5s is what covers that copy.
|
||||
const afterClassificationMs =
|
||||
LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS - WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS
|
||||
|
||||
expect(WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS).toBeLessThan(
|
||||
LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS
|
||||
)
|
||||
expect(afterClassificationMs).toBeGreaterThanOrEqual(22_000)
|
||||
})
|
||||
|
||||
it('leaves the evidence read enough clock to be worth attempting', () => {
|
||||
// Not a reservation: the probes spend first and this is checked afterwards. Assert only
|
||||
// that the threshold covers what the two steps actually cost, or the launcher would start
|
||||
// a read it cannot finish.
|
||||
const evidenceMs = POSIX_OWNERSHIP_PROBE_DEADLINE_MS * PTY_OWNERSHIP_PROBE_ATTEMPTS
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
// Neither guarded step runs on Windows — there is no session-leader signal to read.
|
||||
expect(CLASSIFICATION_EVIDENCE_MIN_MS).toBe(0)
|
||||
} else {
|
||||
expect(CLASSIFICATION_EVIDENCE_MIN_MS).toBeGreaterThanOrEqual(
|
||||
PS_IDENTITY_TIMEOUT_MS + evidenceMs
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('only attempts the evidence read while the clock can still finish it', () => {
|
||||
// The gate is what keeps an opportunistic read from becoming an overrun: the read costs an
|
||||
// identity ps plus two ownership probes, and it runs after the probes have already spent
|
||||
// whatever they spent. If the threshold ever drops below that cost, a read started near the
|
||||
// ceiling finishes past it — and the ceiling is what the kill ladder and fork are sized
|
||||
// against.
|
||||
const evidenceCostMs =
|
||||
PS_IDENTITY_TIMEOUT_MS + POSIX_OWNERSHIP_PROBE_DEADLINE_MS * PTY_OWNERSHIP_PROBE_ATTEMPTS
|
||||
|
||||
if (process.platform !== 'win32') {
|
||||
expect(CLASSIFICATION_EVIDENCE_MIN_MS).toBeGreaterThanOrEqual(evidenceCostMs)
|
||||
}
|
||||
// And the ceiling must still hold if a read starts at the very last moment the gate allows.
|
||||
expect(WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS).toBeGreaterThanOrEqual(
|
||||
CLASSIFICATION_EVIDENCE_MIN_MS
|
||||
)
|
||||
})
|
||||
|
||||
it('gives the patient ask more clock than the cheap ask it replaced', () => {
|
||||
// Kept as arithmetic, but it is NOT the guard: this restates the expression rather than
|
||||
// executing it, so it cannot catch the expression being replaced. daemon-init.test.ts
|
||||
// 'spends a patient connect budget on the wedged ask' watches the launcher actually spend
|
||||
// it, and is the test that fails when this collapses back to the cheap constant.
|
||||
const elapsedBeforeAsk = OCCUPANCY_CONNECT_BUDGET_MS + HEALTH_CHECK_TIMEOUT_MS
|
||||
const probeBudgetMs = WEDGED_DAEMON_CLASSIFICATION_BUDGET_MS - elapsedBeforeAsk
|
||||
const patientConnectMs = Math.max(
|
||||
OCCUPANCY_CONNECT_BUDGET_MS,
|
||||
probeBudgetMs - OCCUPANCY_REQUEST_BUDGET_MS
|
||||
)
|
||||
|
||||
expect(patientConnectMs).toBeGreaterThan(OCCUPANCY_CONNECT_BUDGET_MS)
|
||||
})
|
||||
})
|
||||
@@ -1,437 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { inspectDaemonPtyOwnership } from './daemon-live-pty-evidence'
|
||||
import type { ProcessTableRow } from '../../shared/process-table-snapshot'
|
||||
|
||||
const { readFreshProcessTable, readCachedProcessTable } = vi.hoisted(() => ({
|
||||
readFreshProcessTable: vi.fn(async () => [] as ProcessTableRow[]),
|
||||
readCachedProcessTable: vi.fn(async () => [] as ProcessTableRow[])
|
||||
}))
|
||||
|
||||
// Spread the original: the Windows enumerator builds its reader from this module too.
|
||||
vi.mock('../../shared/process-table-snapshot', async (importOriginal) => ({
|
||||
...(await importOriginal<Record<string, unknown>>()),
|
||||
getFreshProcessTableSnapshot: readFreshProcessTable,
|
||||
getProcessTableSnapshot: readCachedProcessTable
|
||||
}))
|
||||
|
||||
const DAEMON_PID = 4242
|
||||
|
||||
function row(pid: number, ppid: number, overrides: Partial<ProcessTableRow> = {}): ProcessTableRow {
|
||||
return { pid, ppid, stat: 'Ss', command: '/bin/bash', ...overrides }
|
||||
}
|
||||
|
||||
const daemonRow = row(DAEMON_PID, 1, { command: 'daemon-entry.js' })
|
||||
|
||||
// macOS wraps every terminal in login(1); only the wrapper is the session leader.
|
||||
const LOGIN_WRAPPER = '/usr/bin/login -flpq nwparker /bin/bash …'
|
||||
|
||||
function posixTable(rows: ProcessTableRow[]): () => Promise<ProcessTableRow[]> {
|
||||
return async () => rows
|
||||
}
|
||||
|
||||
describe('inspectDaemonPtyOwnership on POSIX', () => {
|
||||
it.each(['darwin', 'linux'] as const)('reports live PTY ownership on %s', async (platform) => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform,
|
||||
readPosixProcessTable: posixTable([daemonRow, row(101, DAEMON_PID)])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('counts a grandchild, since macOS wraps every shell in login(1)', async () => {
|
||||
// daemon -> login(1) -> shell: a direct-children test would miss the agent entirely.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { command: '/usr/bin/login -flpq nwparker' }),
|
||||
row(202, 101, { stat: 'S+', command: 'claude' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('reports no live PTYs for an observed root with no descendants', async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([daemonRow, row(999, 1)])
|
||||
})
|
||||
).resolves.toBe('no-live-ptys')
|
||||
})
|
||||
|
||||
it('does not count zombies, which a wedged daemon cannot reap', async () => {
|
||||
// Why this matters: the daemon is wedged precisely because its event loop is blocked,
|
||||
// so every already-exited agent lingers as <defunct>. Counting them would read
|
||||
// "all agents finished" as "agents still running" — correlated with the wedge itself.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'Z+', command: '<defunct>' }),
|
||||
row(102, DAEMON_PID, { stat: 'Z', command: '<defunct>' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('no-live-ptys')
|
||||
})
|
||||
|
||||
it('still counts a live descendant hidden behind a zombie parent', async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'Z', command: '<defunct>' }),
|
||||
row(202, 101, { stat: 'Ss', command: 'codex' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('ignores helpers the daemon forked, which are not session leaders', async () => {
|
||||
// Why this and not re-sampling: a hung `scutil`, credential helper or PTY-spawn health
|
||||
// check outlives any sampling gap — often it is *why* the daemon is wedged. Only a PTY
|
||||
// child is a session leader (forkpty calls setsid), so the flag is the real discriminator.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'S', command: '/usr/sbin/scutil --dns' }),
|
||||
row(102, DAEMON_PID, { stat: 'R+', command: '/bin/sh -c exit 0' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('no-live-ptys')
|
||||
})
|
||||
|
||||
it("excludes the daemon's own PTY-spawn probe, which forkpty also makes a session leader", async () => {
|
||||
// Why the stat flag is not enough: the daemon opens this PTY itself, so a daemon hosting
|
||||
// zero user terminals would be held forever on the strength of its own stuck health check.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'Ss', command: '/bin/sh -c exit 0' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('no-live-ptys')
|
||||
})
|
||||
|
||||
it("still counts a real terminal sitting beside the daemon's own probe", async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'Ss', command: '/bin/sh -c exit 0' }),
|
||||
row(202, DAEMON_PID, { stat: 'Ss+', command: 'claude' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('does not exclude an agent whose command merely contains a probe command', async () => {
|
||||
// Exact match, not prefix or substring: `sh -c` payloads are user-supplied, and treating one
|
||||
// as the daemon's own probe discards proof that killing the daemon would end real work.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'Ss', command: '/bin/sh -c exit 0 && claude' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('counts a session leader reached through a non-session-leader hop', async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { stat: 'S', command: 'wrapper' }),
|
||||
row(202, 101, { stat: 'Ss+', command: 'claude' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('ignores a login wrapper stranded without its shell (#13764)', async () => {
|
||||
// Why: the macOS TCC wrapper can outlive the shell it wrapped, leaving a session leader
|
||||
// hosting nothing. On hosts where those accumulate, counting them would hold a daemon
|
||||
// whose sessions have all ended — indefinitely, and for no live work.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { command: '/usr/bin/login -flpq nwparker /bin/bash …' }),
|
||||
row(102, DAEMON_PID, { command: '/usr/bin/login -flpq nwparker /bin/bash …' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('no-live-ptys')
|
||||
})
|
||||
|
||||
it('still counts a login wrapper that has its shell', async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { command: '/usr/bin/login -flpq nwparker /bin/bash …' }),
|
||||
row(202, 101, { command: '/opt/homebrew/bin/bash --rcfile …' })
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('reports unknown when the table never contained the daemon', async () => {
|
||||
// Why: an unobserved root yields the same empty result as a childless one —
|
||||
// reading that as "empty" authorizes killing a daemon full of live agents.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([row(999, 1)])
|
||||
})
|
||||
).resolves.toBe('unknown')
|
||||
})
|
||||
|
||||
it('falls back to the cached table when the uncached read blows its deadline', async () => {
|
||||
// Why this matters most on the busiest host: every agent pane drives the shared reader on
|
||||
// its own cadence, so the uncached read queues behind them and can expire on queueing
|
||||
// alone — going blind exactly where the daemon has the most agents to lose.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
posixDeadlineMs: 5,
|
||||
readPosixProcessTable: () => new Promise<ProcessTableRow[]>(() => {}),
|
||||
readCachedPosixProcessTable: posixTable([daemonRow, row(101, DAEMON_PID)])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('reports unknown only when the cached table is blind too', async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
posixDeadlineMs: 5,
|
||||
readPosixProcessTable: () => new Promise<ProcessTableRow[]>(() => {}),
|
||||
readCachedPosixProcessTable: () => new Promise<ProcessTableRow[]>(() => {})
|
||||
})
|
||||
).resolves.toBe('unknown')
|
||||
})
|
||||
|
||||
it('reports unknown when the process table cannot be read', async () => {
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: async () => {
|
||||
throw new Error('ps timed out')
|
||||
}
|
||||
})
|
||||
).resolves.toBe('unknown')
|
||||
})
|
||||
|
||||
it('tolerates a ppid cycle reachable from the daemon without hanging', async () => {
|
||||
// Why this shape: `ps` is not atomic, so a re-parented process can appear twice and
|
||||
// close a loop. The cycle must be reachable from the root or the walk never enters it.
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: posixTable([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID),
|
||||
row(102, 101),
|
||||
row(101, 102)
|
||||
])
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
})
|
||||
|
||||
describe('inspectDaemonPtyOwnership on win32', () => {
|
||||
it('abstains rather than counting descendants it cannot classify', async () => {
|
||||
// Why no verdict at all: the POSIX signal is that a hosted terminal is a session leader,
|
||||
// and Windows has no equivalent — so the only available answer was "any descendant",
|
||||
// which counts the orphaned conpty hosts a wedged daemon cannot reap. Holding on those
|
||||
// would make a wedged, empty daemon unreplaceable forever. 'unknown' leaves Windows as it
|
||||
// was before this change instead of trading one failure mode for a worse one.
|
||||
const readPosixProcessTable = vi.fn(async () => [daemonRow, row(101, DAEMON_PID)])
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'win32', readPosixProcessTable })
|
||||
).resolves.toBe('unknown')
|
||||
expect(readPosixProcessTable).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('inspectDaemonPtyOwnership login(1) handling', () => {
|
||||
it('counts a childless login(1) as live work off darwin', async () => {
|
||||
// Orca only wraps terminals in login(1) on macOS, so elsewhere this pattern is the user's
|
||||
// own login — and one still prompting for credentials has no child yet. Excluding it there
|
||||
// discards real work to solve a macOS problem.
|
||||
const rows = [
|
||||
daemonRow,
|
||||
row(5000, DAEMON_PID, { stat: 'Ss', command: '/usr/bin/login nwparker' })
|
||||
]
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'linux',
|
||||
readPosixProcessTable: async () => rows
|
||||
})
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('still excludes a childless login(1) on darwin (#13764)', async () => {
|
||||
const rows = [
|
||||
daemonRow,
|
||||
row(5000, DAEMON_PID, { stat: 'Ss', command: '/usr/bin/login -pf nwparker /bin/zsh' })
|
||||
]
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
readPosixProcessTable: async () => rows
|
||||
})
|
||||
).resolves.toBe('no-live-ptys')
|
||||
})
|
||||
})
|
||||
|
||||
describe('inspectDaemonPtyOwnership sampling', () => {
|
||||
it('will not confirm emptiness from the cached table the first read already used', async () => {
|
||||
// Two agreeing samples are only worth more than one if they are two observations. When the
|
||||
// fresh read is slow — the busy host this evidence exists for — both attempts fell through
|
||||
// to the same TTL-cached snapshot, so a login(1) wrapper photographed before its shell
|
||||
// appeared could be 'confirmed' empty by a second look at the same photograph.
|
||||
const readCachedPosixProcessTable = vi.fn<() => Promise<ProcessTableRow[]>>(async () => [
|
||||
daemonRow
|
||||
])
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
posixDeadlineMs: 5,
|
||||
// Never settles inside the deadline, so every attempt reaches for the cache.
|
||||
readPosixProcessTable: () => new Promise<ProcessTableRow[]>(() => {}),
|
||||
readCachedPosixProcessTable
|
||||
})
|
||||
).resolves.toBe('unknown')
|
||||
|
||||
// Only the first sample may be served from the cache; the confirming one must not be.
|
||||
expect(readCachedPosixProcessTable).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('will not let a blind confirming read turn emptiness into a verdict', async () => {
|
||||
// Emptiness authorizes a kill, so it needs corroboration. A read that saw nothing at all
|
||||
// corroborates nothing — treating it as agreement is the step this module refuses.
|
||||
const readPosixProcessTable = vi
|
||||
.fn<() => Promise<ProcessTableRow[]>>()
|
||||
.mockResolvedValueOnce([daemonRow])
|
||||
.mockRejectedValueOnce(new Error('ps timed out'))
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, {
|
||||
platform: 'darwin',
|
||||
posixDeadlineMs: 5,
|
||||
readPosixProcessTable,
|
||||
readCachedPosixProcessTable: async () => {
|
||||
throw new Error('cached read blind too')
|
||||
}
|
||||
})
|
||||
).resolves.toBe('unknown')
|
||||
})
|
||||
it('takes a conclusive answer on the first read, without re-sampling', async () => {
|
||||
const readPosixProcessTable = vi.fn(async () => [daemonRow, row(101, DAEMON_PID)])
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin', readPosixProcessTable })
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
expect(readPosixProcessTable).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('retries a blind read, because the load that wedges the daemon also blinds ps', async () => {
|
||||
const readPosixProcessTable = vi
|
||||
.fn<() => Promise<ProcessTableRow[]>>()
|
||||
.mockRejectedValueOnce(new Error('ps timed out'))
|
||||
.mockResolvedValueOnce([daemonRow, row(101, DAEMON_PID)])
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin', readPosixProcessTable })
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('preserves on a sighting the second read could not contradict', async () => {
|
||||
// Why: a blind read is not evidence against a live one. Killing agents is unrecoverable.
|
||||
const readPosixProcessTable = vi
|
||||
.fn<() => Promise<ProcessTableRow[]>>()
|
||||
.mockResolvedValueOnce([daemonRow, row(101, DAEMON_PID)])
|
||||
.mockRejectedValueOnce(new Error('ps timed out'))
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin', readPosixProcessTable })
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('confirms emptiness with a second read before letting it authorize a kill', async () => {
|
||||
const readPosixProcessTable = vi.fn(async () => [daemonRow])
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin', readPosixProcessTable })
|
||||
).resolves.toBe('no-live-ptys')
|
||||
expect(readPosixProcessTable).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('sees a terminal whose shell had not yet appeared on the first read', async () => {
|
||||
// A terminal contributes exactly one session leader — on macOS the login wrapper — and it
|
||||
// is childless for the moment between forkpty creating it and the shell appearing. One
|
||||
// snapshot cannot tell that from a wrapper whose shell has gone, and guessing wrong here
|
||||
// kills a live terminal.
|
||||
const readPosixProcessTable = vi
|
||||
.fn<() => Promise<ProcessTableRow[]>>()
|
||||
.mockResolvedValueOnce([daemonRow, row(101, DAEMON_PID, { command: LOGIN_WRAPPER })])
|
||||
.mockResolvedValueOnce([
|
||||
daemonRow,
|
||||
row(101, DAEMON_PID, { command: LOGIN_WRAPPER }),
|
||||
row(202, 101, { stat: 'S+', command: '/opt/homebrew/bin/bash --rcfile …' })
|
||||
])
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin', readPosixProcessTable })
|
||||
).resolves.toBe('owns-live-ptys')
|
||||
})
|
||||
|
||||
it('gives up as unknown rather than guessing when every read stays blind', async () => {
|
||||
const readPosixProcessTable = vi.fn(async (): Promise<ProcessTableRow[]> => {
|
||||
throw new Error('ps timed out')
|
||||
})
|
||||
|
||||
await expect(
|
||||
inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin', readPosixProcessTable })
|
||||
).resolves.toBe('unknown')
|
||||
expect(readPosixProcessTable.mock.calls.length).toBeGreaterThan(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('inspectDaemonPtyOwnership POSIX process-table source', () => {
|
||||
beforeEach(() => {
|
||||
readFreshProcessTable.mockReset()
|
||||
readCachedProcessTable.mockReset()
|
||||
readFreshProcessTable.mockResolvedValue([daemonRow, row(101, DAEMON_PID)])
|
||||
readCachedProcessTable.mockResolvedValue([])
|
||||
})
|
||||
|
||||
it('reads an uncached table, since the cached one can predate the PTYs it protects', async () => {
|
||||
// The 500ms TTL would also hand both samples the same array, collapsing the confirmation.
|
||||
await expect(inspectDaemonPtyOwnership(DAEMON_PID, { platform: 'darwin' })).resolves.toBe(
|
||||
'owns-live-ptys'
|
||||
)
|
||||
expect(readFreshProcessTable).toHaveBeenCalled()
|
||||
expect(readCachedProcessTable).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -1,260 +0,0 @@
|
||||
import {
|
||||
getFreshProcessTableSnapshot,
|
||||
getProcessTableSnapshot,
|
||||
type ProcessTableRow
|
||||
} from '../../shared/process-table-snapshot'
|
||||
|
||||
/**
|
||||
* Out-of-band answer to "is this daemon still hosting running terminals?".
|
||||
*
|
||||
* Deliberately never touches the daemon socket: the only caller asks precisely
|
||||
* because the daemon has already failed to answer over it, and a wedged daemon
|
||||
* cannot be asked to vouch for its own sessions. Replacing a daemon kills every
|
||||
* process it hosts, so that decision needs evidence that survives the wedge.
|
||||
*
|
||||
* 'unknown' is not "empty" — it means the process table could not be read, or did not
|
||||
* contain the daemon at all. It is not evidence of absence, and it is deliberately not
|
||||
* evidence of presence either: the only caller raises to preserve on 'owns-live-ptys' alone.
|
||||
*
|
||||
* What 'unknown' costs changed with the launcher: it no longer falls through to a kill, it
|
||||
* holds the daemon in degraded mode. So this module's job is now to spare the user that
|
||||
* degradation where it safely can, not to stand between them and a dead agent.
|
||||
*/
|
||||
export type DaemonPtyOwnership = 'owns-live-ptys' | 'no-live-ptys' | 'unknown'
|
||||
|
||||
export type DaemonPtyOwnershipDeps = {
|
||||
platform?: NodeJS.Platform
|
||||
readPosixProcessTable?: () => Promise<ProcessTableRow[]>
|
||||
readCachedPosixProcessTable?: () => Promise<ProcessTableRow[]>
|
||||
posixDeadlineMs?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Why sampled twice: the load that wedges the daemon is the same load that can blind
|
||||
* the process-table read, so a single blind sample would lose the evidence exactly when
|
||||
* it matters most. Only blindness is retried — a conclusive answer is taken as given.
|
||||
* This runs only on the replace path, after ~60s of grace is already spent.
|
||||
*/
|
||||
export const PTY_OWNERSHIP_PROBE_ATTEMPTS = 2
|
||||
|
||||
/**
|
||||
* POSIX needs its own ceiling for the same reason: the shared reader's `ps` timeout does not
|
||||
* cover queueing behind an in-flight scan, and this runs on a launch that fails open.
|
||||
*/
|
||||
export const POSIX_OWNERSHIP_PROBE_DEADLINE_MS = 4_000
|
||||
|
||||
/** macos-tcc-login-shell.ts wraps every darwin terminal in this. */
|
||||
const MACOS_LOGIN_WRAPPER_PREFIX = '/usr/bin/login '
|
||||
|
||||
function withDeadline<T>(work: Promise<T>, deadlineMs: number, onDeadline: T): Promise<T> {
|
||||
return new Promise<T>((resolve) => {
|
||||
const timer = setTimeout(() => resolve(onDeadline), deadlineMs)
|
||||
timer.unref?.()
|
||||
void work.then(
|
||||
(value) => {
|
||||
clearTimeout(timer)
|
||||
resolve(value)
|
||||
},
|
||||
() => {
|
||||
clearTimeout(timer)
|
||||
resolve(onDeadline)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* The daemon opens PTYs for its own probes, and forkpty makes those session leaders too, so
|
||||
* process state alone cannot tell them from a hosted terminal. Each is a fixed, argument-less
|
||||
* command the daemon issues itself, so matching them exactly costs no real terminal.
|
||||
*/
|
||||
const DAEMON_SELF_SPAWNED_PTY_COMMANDS = [
|
||||
// pty-subprocess.ts checkPtySpawnHealth
|
||||
{ program: 'sh', args: '-c exit 0' }
|
||||
]
|
||||
|
||||
function isDaemonSelfSpawnedPty(row: Pick<ProcessTableRow, 'command'>): boolean {
|
||||
const command = row.command.trim()
|
||||
return DAEMON_SELF_SPAWNED_PTY_COMMANDS.some(({ program, args }) => {
|
||||
const suffix = ` ${args}`
|
||||
// Why the argv tail must match exactly: `sh -c` payloads are user-supplied, and treating
|
||||
// one as the daemon's own probe would discard proof that real work is running.
|
||||
if (!command.endsWith(suffix)) {
|
||||
return false
|
||||
}
|
||||
// Only the program may vary, and only by path, so compare its trailing segment rather
|
||||
// than the whole string.
|
||||
const executable = command.slice(0, command.length - suffix.length)
|
||||
return (executable.split('/').pop() ?? executable) === program
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* A PTY child is a session leader — forkpty() calls setsid() — which the daemon's plain
|
||||
* subprocesses (a `scutil` resolver probe, a stuck credential helper) never are.
|
||||
*
|
||||
* Zombies are excluded for a correlated reason: a wedged daemon cannot reap, so its
|
||||
* already-exited PTYs linger as <defunct> and would read as still running.
|
||||
*/
|
||||
function isLivePtySessionLeader(row: ProcessTableRow): boolean {
|
||||
// Lowercase 's' is only ever the session-leader flag; no process state code uses it.
|
||||
return !row.stat.startsWith('Z') && row.stat.includes('s') && !isDaemonSelfSpawnedPty(row)
|
||||
}
|
||||
|
||||
/**
|
||||
* macOS wraps every terminal in `/usr/bin/login` for TCC attribution, and the wrapper can
|
||||
* outlive the shell it wrapped (#13764) — a session leader hosting nothing. Counting those
|
||||
* would hold a daemon whose sessions have all ended, on hosts where they accumulate by the
|
||||
* hundred. A wrapper still doing its job always has the shell it exec'd beneath it.
|
||||
*/
|
||||
function isStrandedLoginWrapper(
|
||||
row: ProcessTableRow,
|
||||
hasChildren: boolean,
|
||||
platform: NodeJS.Platform
|
||||
): boolean {
|
||||
// Why darwin only: Orca wraps terminals in login(1) for TCC attribution on macOS and nowhere
|
||||
// else, so off darwin this pattern can only ever be a user's own login(1) — and one that is
|
||||
// still prompting for credentials has no child yet, which is exactly the shape excluded here.
|
||||
// Applied POSIX-wide it discarded real work to solve a macOS problem.
|
||||
return (
|
||||
platform === 'darwin' &&
|
||||
!hasChildren &&
|
||||
row.command.trim().startsWith(MACOS_LOGIN_WRAPPER_PREFIX)
|
||||
)
|
||||
}
|
||||
|
||||
function collectLivePtyDescendants(
|
||||
rows: ProcessTableRow[],
|
||||
rootPid: number,
|
||||
platform: NodeJS.Platform
|
||||
): ProcessTableRow[] {
|
||||
const childrenByPpid = new Map<number, ProcessTableRow[]>()
|
||||
for (const row of rows) {
|
||||
if (row.pid === rootPid) {
|
||||
continue
|
||||
}
|
||||
const siblings = childrenByPpid.get(row.ppid)
|
||||
if (siblings) {
|
||||
siblings.push(row)
|
||||
} else {
|
||||
childrenByPpid.set(row.ppid, [row])
|
||||
}
|
||||
}
|
||||
const visited = new Set<number>([rootPid])
|
||||
const queue: number[] = [rootPid]
|
||||
const live: ProcessTableRow[] = []
|
||||
while (queue.length > 0) {
|
||||
const pid = queue.shift() as number
|
||||
for (const child of childrenByPpid.get(pid) ?? []) {
|
||||
if (visited.has(child.pid)) {
|
||||
continue
|
||||
}
|
||||
visited.add(child.pid)
|
||||
queue.push(child.pid)
|
||||
if (
|
||||
isLivePtySessionLeader(child) &&
|
||||
!isStrandedLoginWrapper(child, childrenByPpid.has(child.pid), platform)
|
||||
) {
|
||||
live.push(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
return live
|
||||
}
|
||||
|
||||
/** POSIX only; Windows abstains before this is reached. */
|
||||
async function probeOnce(
|
||||
daemonPid: number,
|
||||
deps: DaemonPtyOwnershipDeps,
|
||||
/**
|
||||
* The cached table may stand in for a slow read when the answer we are protecting is
|
||||
* 'owns-live-ptys', but never when confirming emptiness: a confirmation drawn from the same
|
||||
* TTL-cached snapshot as the sample it confirms is one observation counted twice, and the
|
||||
* window it is meant to exclude — a login(1) wrapper whose shell has not appeared yet — is
|
||||
* shorter than the cache. Denied there, a slow read answers 'unknown', which holds.
|
||||
*/
|
||||
allowCachedFallback = true
|
||||
): Promise<DaemonPtyOwnership> {
|
||||
const deadlineMs = deps.posixDeadlineMs ?? POSIX_OWNERSHIP_PROBE_DEADLINE_MS
|
||||
const deadline = Date.now() + deadlineMs
|
||||
const remaining = (): number => Math.max(1, deadline - Date.now())
|
||||
const rows =
|
||||
(await withDeadline(
|
||||
(deps.readPosixProcessTable ?? getFreshProcessTableSnapshot)(),
|
||||
remaining(),
|
||||
null
|
||||
)) ??
|
||||
// Why fall back instead of answering 'unknown': the uncached reader queues behind the
|
||||
// scans every agent pane already drives, so the busiest host — the one this evidence
|
||||
// exists to protect — is the likeliest to blow the deadline on queueing alone. A table a
|
||||
// few hundred milliseconds old still shows whether this daemon has children, and going
|
||||
// blind here gets them killed. It shares the attempt's budget rather than doubling it,
|
||||
// so an attempt costs what the launch budget was told it costs.
|
||||
(allowCachedFallback
|
||||
? await withDeadline(
|
||||
(deps.readCachedPosixProcessTable ?? getProcessTableSnapshot)(),
|
||||
remaining(),
|
||||
null
|
||||
)
|
||||
: null)
|
||||
// Why: a walk that never saw the root reports zero descendants for a process it
|
||||
// never examined. Only a root we actually observed can prove emptiness — and a read
|
||||
// that blew its deadline saw nothing at all.
|
||||
if (rows === null || !rows.some((row) => row.pid === daemonPid)) {
|
||||
return 'unknown'
|
||||
}
|
||||
return collectLivePtyDescendants(rows, daemonPid, deps.platform ?? process.platform).length > 0
|
||||
? 'owns-live-ptys'
|
||||
: 'no-live-ptys'
|
||||
}
|
||||
|
||||
/**
|
||||
* A session-leader descendant is positive proof that killing this daemon would destroy
|
||||
* running work. Descendants rather than direct children: macOS wraps every shell in
|
||||
* login(1) for TCC attribution, so the agent is a grandchild at best.
|
||||
*/
|
||||
export async function inspectDaemonPtyOwnership(
|
||||
daemonPid: number,
|
||||
deps: DaemonPtyOwnershipDeps = {}
|
||||
): Promise<DaemonPtyOwnership> {
|
||||
const platform = deps.platform ?? process.platform
|
||||
// Why Windows gets no verdict at all: the POSIX signal is a property only a hosted terminal
|
||||
// has — forkpty makes it a session leader — and Windows has no equivalent, so the branch
|
||||
// that lived here could only count descendants. That reads a wedged daemon's orphaned
|
||||
// conpty hosts as live work, since ClosePseudoConsole runs on the daemon's own JS thread
|
||||
// and a daemon too wedged to answer is too wedged to reap them.
|
||||
//
|
||||
// Abstaining costs Windows nothing it had: this evidence can only ever raise 'unknown' to
|
||||
// 'occupied', and both already hold the daemon. A verdict here would only let Windows print
|
||||
// the more accurate of two identical outcomes, which is not worth guessing for.
|
||||
if (platform === 'win32') {
|
||||
return 'unknown'
|
||||
}
|
||||
let emptyAwaitingConfirmation = false
|
||||
for (let attempt = 0; attempt < PTY_OWNERSHIP_PROBE_ATTEMPTS; attempt++) {
|
||||
let sample: DaemonPtyOwnership
|
||||
try {
|
||||
// The confirming read must be its own observation, so it is denied the cached table.
|
||||
sample = await probeOnce(daemonPid, deps, !emptyAwaitingConfirmation)
|
||||
} catch {
|
||||
sample = 'unknown'
|
||||
}
|
||||
// Why a second look before accepting emptiness: a terminal contributes exactly
|
||||
// one session leader — on macOS the login wrapper — and it is invisible for the moment
|
||||
// between forkpty creating it and the shell appearing beneath it. One snapshot cannot tell
|
||||
// that from a wrapper whose shell has gone. Emptiness authorizes a kill, so it is the
|
||||
// answer worth paying a second read for; 'owns-live-ptys' needs no confirmation.
|
||||
if (sample === 'no-live-ptys' && !emptyAwaitingConfirmation) {
|
||||
emptyAwaitingConfirmation = true
|
||||
continue
|
||||
}
|
||||
if (sample !== 'unknown') {
|
||||
return sample
|
||||
}
|
||||
}
|
||||
// Why not 'no-live-ptys' here: reaching this means the confirming read went blind, and a
|
||||
// blind read cannot corroborate anything. Upgrading an unconfirmed emptiness to a definitive
|
||||
// one is exactly the "absence of proof is proof of absence" step this module exists to
|
||||
// refuse — and emptiness is the answer that authorizes a kill.
|
||||
return 'unknown'
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
OCCUPANCY_CONNECT_BUDGET_MS,
|
||||
OCCUPANCY_REQUEST_BUDGET_MS,
|
||||
raiseOccupancyWithProcessEvidence,
|
||||
resolveDaemonOccupancy,
|
||||
type DaemonOccupancy,
|
||||
type DaemonOccupancyDeps
|
||||
} from './daemon-occupancy'
|
||||
import type { inspectDaemonPtyOwnership } from './daemon-live-pty-evidence'
|
||||
|
||||
const SOCKET_PATH = '/tmp/orca-daemon.sock'
|
||||
const TOKEN_PATH = '/tmp/orca-daemon.token'
|
||||
const DAEMON_PID = 4242
|
||||
|
||||
type Ownership = Awaited<ReturnType<typeof inspectDaemonPtyOwnership>>
|
||||
|
||||
function ipcAnswers(count: number | null) {
|
||||
return vi.fn<NonNullable<DaemonOccupancyDeps['listSessions']>>(async () => count)
|
||||
}
|
||||
|
||||
function ownershipIs(ownership: Ownership) {
|
||||
return vi.fn<typeof inspectDaemonPtyOwnership>(async () => ownership)
|
||||
}
|
||||
|
||||
function resolve(
|
||||
deps: DaemonOccupancyDeps,
|
||||
recordedPid: number | null = DAEMON_PID
|
||||
): Promise<Awaited<ReturnType<typeof resolveDaemonOccupancy>>> {
|
||||
return resolveDaemonOccupancy({
|
||||
socketPath: SOCKET_PATH,
|
||||
tokenPath: TOKEN_PATH,
|
||||
recordedPid,
|
||||
deps
|
||||
})
|
||||
}
|
||||
|
||||
describe('resolveDaemonOccupancy with a daemon that answered', () => {
|
||||
it('reports occupied with the counted sessions, without consulting the process table', async () => {
|
||||
const listSessions = ipcAnswers(3)
|
||||
const inspectPtyOwnership = ownershipIs('no-live-ptys')
|
||||
|
||||
await expect(resolve({ listSessions, inspectPtyOwnership })).resolves.toEqual({
|
||||
state: 'occupied',
|
||||
liveSessions: 3
|
||||
})
|
||||
expect(listSessions).toHaveBeenCalledWith(
|
||||
SOCKET_PATH,
|
||||
TOKEN_PATH,
|
||||
expect.any(Number),
|
||||
expect.any(Number)
|
||||
)
|
||||
// The daemon's own reply is authoritative; process-table evidence could only muddy it.
|
||||
expect(inspectPtyOwnership).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports empty on a count of zero, without consulting the process table', async () => {
|
||||
// The one state that licenses a kill, and only the daemon itself can establish it.
|
||||
const listSessions = ipcAnswers(0)
|
||||
const inspectPtyOwnership = ownershipIs('owns-live-ptys')
|
||||
|
||||
await expect(resolve({ listSessions, inspectPtyOwnership })).resolves.toEqual({
|
||||
state: 'empty',
|
||||
liveSessions: 0
|
||||
})
|
||||
expect(inspectPtyOwnership).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports occupied for a single session', async () => {
|
||||
await expect(
|
||||
resolve({ listSessions: ipcAnswers(1), inspectPtyOwnership: ownershipIs('unknown') })
|
||||
).resolves.toEqual({ state: 'occupied', liveSessions: 1 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveDaemonOccupancy when the daemon could not answer', () => {
|
||||
it('raises to occupied on process-table evidence, keyed to the recorded pid', async () => {
|
||||
const inspectPtyOwnership = ownershipIs('owns-live-ptys')
|
||||
|
||||
await expect(resolve({ listSessions: ipcAnswers(null), inspectPtyOwnership })).resolves.toEqual(
|
||||
{
|
||||
state: 'occupied',
|
||||
liveSessions: null
|
||||
}
|
||||
)
|
||||
expect(inspectPtyOwnership).toHaveBeenCalledWith(DAEMON_PID)
|
||||
})
|
||||
|
||||
it('stays unknown — never empty — when the process table shows no live PTYs', async () => {
|
||||
// The asymmetry the module exists for: the table may only ever *raise* the answer.
|
||||
// A daemon too wedged to list its sessions is exactly as likely to be hosting them,
|
||||
// and ps can miss PTYs it never observed. Reading this as 'empty' would license
|
||||
// killing live agents unrecoverably; 'unknown' is the residual, not permission.
|
||||
const inspectPtyOwnership = ownershipIs('no-live-ptys')
|
||||
|
||||
await expect(resolve({ listSessions: ipcAnswers(null), inspectPtyOwnership })).resolves.toEqual(
|
||||
{
|
||||
state: 'unknown',
|
||||
liveSessions: null
|
||||
}
|
||||
)
|
||||
expect(inspectPtyOwnership).toHaveBeenCalledWith(DAEMON_PID)
|
||||
})
|
||||
|
||||
it('stays unknown when the process table could not be read', async () => {
|
||||
await expect(
|
||||
resolve({ listSessions: ipcAnswers(null), inspectPtyOwnership: ownershipIs('unknown') })
|
||||
).resolves.toEqual({ state: 'unknown', liveSessions: null })
|
||||
})
|
||||
|
||||
it('stays unknown without inspecting an unverified pid', async () => {
|
||||
// A pid we could not tie back to this daemon may have been recycled; its children
|
||||
// would be some other process's, and counting them is evidence about the wrong tree.
|
||||
const inspectPtyOwnership = ownershipIs('owns-live-ptys')
|
||||
|
||||
await expect(
|
||||
resolve({ listSessions: ipcAnswers(null), inspectPtyOwnership }, null)
|
||||
).resolves.toEqual({ state: 'unknown', liveSessions: null })
|
||||
expect(inspectPtyOwnership).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveDaemonOccupancy budgets', () => {
|
||||
it('waits longer for an answer than for a handshake', () => {
|
||||
// Why they differ: a daemon that cannot complete a handshake is wedged and worth
|
||||
// re-asking cheaply; one that answered the handshake is demonstrably alive, and its
|
||||
// count settles the question outright. Collapsing both into one tight budget is what
|
||||
// made a slow-but-answering daemon indistinguishable from a dead one.
|
||||
expect(OCCUPANCY_REQUEST_BUDGET_MS).toBeGreaterThan(OCCUPANCY_CONNECT_BUDGET_MS)
|
||||
})
|
||||
|
||||
it('never spends more than the ceiling the caller handed it', async () => {
|
||||
const listSessions = vi.fn(async (_socket: string, _token: string, budgetMs: number) => {
|
||||
expect(budgetMs).toBeLessThanOrEqual(5_000)
|
||||
return null
|
||||
})
|
||||
|
||||
await expect(
|
||||
resolveDaemonOccupancy({
|
||||
socketPath: SOCKET_PATH,
|
||||
tokenPath: TOKEN_PATH,
|
||||
recordedPid: null,
|
||||
budgetMs: 5_000,
|
||||
deps: { listSessions }
|
||||
})
|
||||
).resolves.toEqual({ state: 'unknown', liveSessions: null })
|
||||
expect(listSessions).toHaveBeenCalledWith(SOCKET_PATH, TOKEN_PATH, 5_000, expect.any(Number))
|
||||
})
|
||||
})
|
||||
|
||||
describe('raiseOccupancyWithProcessEvidence', () => {
|
||||
const UNKNOWN: DaemonOccupancy = { state: 'unknown', liveSessions: null }
|
||||
|
||||
it('raises an unanswered verdict to occupied, with no count to report', async () => {
|
||||
const inspectPtyOwnership = ownershipIs('owns-live-ptys')
|
||||
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence(UNKNOWN, DAEMON_PID, { inspectPtyOwnership })
|
||||
).resolves.toEqual({ state: 'occupied', liveSessions: null })
|
||||
expect(inspectPtyOwnership).toHaveBeenCalledWith(DAEMON_PID)
|
||||
})
|
||||
|
||||
it('leaves an unanswered verdict unknown — never empty — when the table shows no live PTYs', async () => {
|
||||
// The whole point of the raise-only contract: ps can miss PTYs it never observed, so an
|
||||
// empty-looking table is not permission to kill a daemon that could not answer for itself.
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence(UNKNOWN, DAEMON_PID, {
|
||||
inspectPtyOwnership: ownershipIs('no-live-ptys')
|
||||
})
|
||||
).resolves.toEqual(UNKNOWN)
|
||||
})
|
||||
|
||||
it('leaves an unanswered verdict unchanged when the table could not be read', async () => {
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence(UNKNOWN, DAEMON_PID, {
|
||||
inspectPtyOwnership: ownershipIs('unknown')
|
||||
})
|
||||
).resolves.toEqual(UNKNOWN)
|
||||
})
|
||||
|
||||
it('does not inspect an unverified pid', async () => {
|
||||
// A pid we could not tie back to this daemon may have been recycled; its children are
|
||||
// evidence about the wrong process tree.
|
||||
const inspectPtyOwnership = ownershipIs('owns-live-ptys')
|
||||
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence(UNKNOWN, null, { inspectPtyOwnership })
|
||||
).resolves.toEqual(UNKNOWN)
|
||||
expect(inspectPtyOwnership).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns an empty verdict untouched, without consulting the process table', async () => {
|
||||
// 'empty' came from the daemon itself and is the one state that licenses a kill. Re-asking
|
||||
// the table could only lower it back to 'unknown', discarding an IPC-proven answer.
|
||||
const inspectPtyOwnership = ownershipIs('owns-live-ptys')
|
||||
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence({ state: 'empty', liveSessions: 0 }, DAEMON_PID, {
|
||||
inspectPtyOwnership
|
||||
})
|
||||
).resolves.toEqual({ state: 'empty', liveSessions: 0 })
|
||||
expect(inspectPtyOwnership).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps the counted sessions of an already-occupied verdict', async () => {
|
||||
// Raising an answered count to the countless 'occupied' would lose what the daemon reported.
|
||||
const inspectPtyOwnership = ownershipIs('no-live-ptys')
|
||||
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence({ state: 'occupied', liveSessions: 3 }, DAEMON_PID, {
|
||||
inspectPtyOwnership
|
||||
})
|
||||
).resolves.toEqual({ state: 'occupied', liveSessions: 3 })
|
||||
expect(inspectPtyOwnership).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('leaves the verdict unchanged when the inspector throws', async () => {
|
||||
await expect(
|
||||
raiseOccupancyWithProcessEvidence(UNKNOWN, DAEMON_PID, {
|
||||
inspectPtyOwnership: vi.fn<typeof inspectDaemonPtyOwnership>(async () => {
|
||||
throw new Error('process table read exploded')
|
||||
})
|
||||
})
|
||||
).resolves.toEqual(UNKNOWN)
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveDaemonOccupancy when an injected dep throws', () => {
|
||||
it('degrades an inspector rejection to unknown', async () => {
|
||||
// Why: a question that could not be asked is exactly what the residual is for. Letting
|
||||
// it escape would route a failed observation into the launch path.
|
||||
const inspectPtyOwnership = vi.fn<typeof inspectDaemonPtyOwnership>(async () => {
|
||||
throw new Error('process table read exploded')
|
||||
})
|
||||
|
||||
await expect(resolve({ listSessions: ipcAnswers(null), inspectPtyOwnership })).resolves.toEqual(
|
||||
{
|
||||
state: 'unknown',
|
||||
liveSessions: null
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('degrades a failing listSessions dep to unknown', async () => {
|
||||
// countLiveSessionsOverIpc catches internally and returns null; an injected dep is
|
||||
// not held to that, so the module guards it.
|
||||
await expect(
|
||||
resolve({
|
||||
listSessions: async () => {
|
||||
throw new Error('socket vanished')
|
||||
},
|
||||
inspectPtyOwnership: ownershipIs('owns-live-ptys')
|
||||
})
|
||||
).resolves.toEqual({ state: 'unknown', liveSessions: null })
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveDaemonOccupancy with a nonsense count', () => {
|
||||
it.each([Number.NaN, -1, 1.5])('refuses to read %p as emptiness', async (counted) => {
|
||||
// 'empty' is the only verdict that licenses a kill, and `counted > 0` reads every one of
|
||||
// these as empty. The dep is injectable, so it is reachable without asking the daemon.
|
||||
await expect(
|
||||
resolve({
|
||||
listSessions: async () => counted,
|
||||
inspectPtyOwnership: ownershipIs('unknown')
|
||||
})
|
||||
).resolves.toEqual({ state: 'unknown', liveSessions: null })
|
||||
})
|
||||
})
|
||||
@@ -1,151 +0,0 @@
|
||||
import { DaemonClient } from './client'
|
||||
import { inspectDaemonPtyOwnership } from './daemon-live-pty-evidence'
|
||||
import { PROTOCOL_VERSION, type ListSessionsResult } from './types'
|
||||
|
||||
/**
|
||||
* How much work a daemon is hosting, and how sure we are.
|
||||
*
|
||||
* The distinction that matters: only the daemon itself can prove it is *empty*.
|
||||
* The OS process table can prove work exists, but a table that shows nothing may
|
||||
* simply have failed to observe it — so it may only ever add protection, never
|
||||
* license a kill. 'unknown' is the residual, and it is not permission.
|
||||
*/
|
||||
export type DaemonOccupancy =
|
||||
| { state: 'occupied'; liveSessions: number | null }
|
||||
| { state: 'empty'; liveSessions: 0 }
|
||||
| { state: 'unknown'; liveSessions: null }
|
||||
|
||||
export type DaemonOccupancyDeps = {
|
||||
listSessions?: (
|
||||
socketPath: string,
|
||||
tokenPath: string,
|
||||
budgetMs: number,
|
||||
connectBudgetMs: number
|
||||
) => Promise<number | null>
|
||||
inspectPtyOwnership?: typeof inspectDaemonPtyOwnership
|
||||
}
|
||||
|
||||
/**
|
||||
* Why two budgets, not one: connecting and answering fail for different reasons and deserve
|
||||
* different patience. A daemon that cannot complete a handshake is wedged, and asking again
|
||||
* shortly is the cheap way to find out whether it recovers — so connect stays tight and the
|
||||
* caller retries it. A daemon that *did* answer the handshake is demonstrably alive, and its
|
||||
* session count is the one thing that settles the question outright, so it is worth waiting
|
||||
* for. Collapsing both into one tight budget is what made a slow-but-answering daemon
|
||||
* indistinguishable from a dead one, and got its agents killed.
|
||||
*/
|
||||
export const OCCUPANCY_CONNECT_BUDGET_MS = 4_000
|
||||
export const OCCUPANCY_REQUEST_BUDGET_MS = 15_000
|
||||
|
||||
/**
|
||||
* Live session count over the daemon's own socket; null when it could not answer.
|
||||
* `budgetMs` caps the whole exchange, so a caller working to a deadline can hand over what
|
||||
* it has left rather than trusting a constant to still fit.
|
||||
*/
|
||||
async function countLiveSessionsOverIpc(
|
||||
socketPath: string,
|
||||
tokenPath: string,
|
||||
budgetMs: number,
|
||||
connectBudgetMs: number
|
||||
): Promise<number | null> {
|
||||
const client = new DaemonClient({ socketPath, tokenPath, protocolVersion: PROTOCOL_VERSION })
|
||||
const deadline = Date.now() + budgetMs
|
||||
const remaining = (): number => Math.max(1, deadline - Date.now())
|
||||
try {
|
||||
await client.ensureConnectedWithin(Math.min(connectBudgetMs, remaining()))
|
||||
const result = await client.request<ListSessionsResult>(
|
||||
'listSessions',
|
||||
undefined,
|
||||
Math.min(OCCUPANCY_REQUEST_BUDGET_MS, remaining())
|
||||
)
|
||||
return result.sessions.filter((session) => session.isAlive).length
|
||||
} catch {
|
||||
return null
|
||||
} finally {
|
||||
client.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the daemon first — a reply is authoritative both ways. Only when it cannot
|
||||
* answer do we fall back to the process table, and then only to *raise* the answer
|
||||
* to 'occupied'. A blind or empty-looking table stays 'unknown', because a daemon
|
||||
* too wedged to list its sessions is exactly as likely to be hosting them.
|
||||
*
|
||||
* `recordedPid` must already be identity-verified, or the evidence could describe
|
||||
* a recycled pid's children rather than this daemon's terminals.
|
||||
*/
|
||||
export async function resolveDaemonOccupancy(args: {
|
||||
socketPath: string
|
||||
tokenPath: string
|
||||
recordedPid: number | null
|
||||
/** Ceiling for the whole resolution; defaults to the connect plus request budgets. */
|
||||
budgetMs?: number
|
||||
/**
|
||||
* How long to spend on the handshake alone. Defaults to the cheap ask. A caller that has
|
||||
* budget left and no answer yet should raise it: retrying a four-second handshake twelve
|
||||
* times cannot reach a daemon that consistently needs five, and this path is only reached
|
||||
* because a three-second health check already timed out — so re-asking on a stricter budget
|
||||
* than the one that triaged it here can only ever agree with it.
|
||||
*/
|
||||
connectBudgetMs?: number
|
||||
deps?: DaemonOccupancyDeps
|
||||
}): Promise<DaemonOccupancy> {
|
||||
const { socketPath, tokenPath, recordedPid, deps = {} } = args
|
||||
const budgetMs = args.budgetMs ?? OCCUPANCY_CONNECT_BUDGET_MS + OCCUPANCY_REQUEST_BUDGET_MS
|
||||
const connectBudgetMs = args.connectBudgetMs ?? OCCUPANCY_CONNECT_BUDGET_MS
|
||||
const unknown: DaemonOccupancy = { state: 'unknown', liveSessions: null }
|
||||
// Why catch rather than let it propagate: 'unknown' is this module's residual, and a
|
||||
// question that could not be asked is the residual's whole purpose. An escaping throw
|
||||
// would route a failed observation into the launch path instead.
|
||||
try {
|
||||
const counted = await (deps.listSessions ?? countLiveSessionsOverIpc)(
|
||||
socketPath,
|
||||
tokenPath,
|
||||
budgetMs,
|
||||
connectBudgetMs
|
||||
)
|
||||
if (counted !== null) {
|
||||
// Why validate a number we just asked for: 'empty' is the one verdict that licenses a
|
||||
// kill, and `counted > 0` quietly reads NaN, -1 and every other non-count as emptiness.
|
||||
// The dep is injectable, so that is reachable without the daemon ever being asked.
|
||||
if (!Number.isInteger(counted) || counted < 0) {
|
||||
return unknown
|
||||
}
|
||||
return counted > 0
|
||||
? { state: 'occupied', liveSessions: counted }
|
||||
: { state: 'empty', liveSessions: 0 }
|
||||
}
|
||||
if (recordedPid === null) {
|
||||
return unknown
|
||||
}
|
||||
const ownership = await (deps.inspectPtyOwnership ?? inspectDaemonPtyOwnership)(recordedPid)
|
||||
return ownership === 'owns-live-ptys' ? { state: 'occupied', liveSessions: null } : unknown
|
||||
} catch {
|
||||
return unknown
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Raise an unanswered verdict with out-of-band evidence. It can only ever raise: an
|
||||
* absent or unreadable process table leaves the verdict exactly as it was, because the
|
||||
* table can prove work exists and never that it does not.
|
||||
*
|
||||
* Separate from the IPC path so a caller waiting for the daemon to recover can re-ask it
|
||||
* cheaply, and pay for the process table once, after the waiting is done.
|
||||
*/
|
||||
export async function raiseOccupancyWithProcessEvidence(
|
||||
occupancy: DaemonOccupancy,
|
||||
recordedPid: number | null,
|
||||
deps: DaemonOccupancyDeps = {}
|
||||
): Promise<DaemonOccupancy> {
|
||||
if (occupancy.state !== 'unknown' || recordedPid === null) {
|
||||
return occupancy
|
||||
}
|
||||
try {
|
||||
const ownership = await (deps.inspectPtyOwnership ?? inspectDaemonPtyOwnership)(recordedPid)
|
||||
return ownership === 'owns-live-ptys' ? { state: 'occupied', liveSessions: null } : occupancy
|
||||
} catch {
|
||||
return occupancy
|
||||
}
|
||||
}
|
||||
@@ -1921,20 +1921,6 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
|
||||
expect(adapter.hasPty(id)).toBe(true)
|
||||
})
|
||||
|
||||
// Why: off-socket the cache can only be missing exits it never received, so a
|
||||
// miss is ignorance. Answering false there authorizes a respawn over a live shell.
|
||||
it('answers unknown, not absent, for an uncached id while disconnected', async () => {
|
||||
const { id } = await adapter.spawn({ cols: 80, rows: 24 })
|
||||
const unknownId = 'repo::/repo/unknown@@deadbeef'
|
||||
expect(adapter.hasPty(unknownId)).toBe(false)
|
||||
|
||||
await server.shutdown()
|
||||
await waitFor(() => !(adapter as unknown as { client: DaemonClient }).client.isConnected())
|
||||
|
||||
expect(adapter.hasPty(unknownId)).toBe(null)
|
||||
expect(adapter.hasPty(id)).toBe(true)
|
||||
})
|
||||
|
||||
it('returns active sessions', async () => {
|
||||
await adapter.spawn({
|
||||
cols: 80,
|
||||
|
||||
@@ -983,10 +983,8 @@ export class DaemonPtyAdapter implements IPtyProvider {
|
||||
return providerSequence ? { providerSequence } : undefined
|
||||
}
|
||||
|
||||
hasPty(id: string): boolean | null {
|
||||
// Why null off-socket: the cache only tracks exits it received, so a miss while
|
||||
// disconnected (or before the first listSessions) is ignorance, not absence.
|
||||
return this.activeSessionIds.has(id) ? true : this.client.isConnected() ? false : null
|
||||
hasPty(id: string): boolean {
|
||||
return this.activeSessionIds.has(id)
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
|
||||
@@ -537,33 +537,6 @@ describe('DaemonPtyRouter', () => {
|
||||
expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session')
|
||||
})
|
||||
|
||||
// Why: an unreachable generation cannot certify that another generation's session
|
||||
// is gone — collapsing its null to false hands a respawn a fabricated death.
|
||||
it('answers unknown when any adapter cannot see the session', () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy')
|
||||
legacy.hasPty = vi.fn(() => null)
|
||||
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
|
||||
|
||||
expect(router.hasPty('unmapped-session')).toBe(null)
|
||||
})
|
||||
|
||||
it('does not report terminal_gone for inspection when an adapter cannot answer', async () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy')
|
||||
legacy.hasPty = vi.fn(() => null)
|
||||
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
|
||||
|
||||
await expect(router.inspectProcess('unmapped-session')).resolves.toEqual({
|
||||
foregroundProcess: null,
|
||||
hasChildProcesses: false
|
||||
})
|
||||
|
||||
// Unproven routes must not be memoized as an owner.
|
||||
const unanimous = new DaemonPtyRouter({ current: createAdapter('c2'), legacy: [] })
|
||||
await expect(unanimous.inspectProcess('unmapped-session')).rejects.toThrow('terminal_gone')
|
||||
})
|
||||
|
||||
it('discovers an unmapped live session from one coalesced inventory', async () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy', ['surviving-session'])
|
||||
|
||||
@@ -77,15 +77,12 @@ export class DaemonPtyRouter implements IPtyProvider {
|
||||
return await this.adapterFor(id).attach(id)
|
||||
}
|
||||
|
||||
hasPty(id: string): boolean | null {
|
||||
hasPty(id: string): boolean {
|
||||
const routed = this.sessionAdapters.get(id)
|
||||
if (routed) {
|
||||
return routed.hasPty(id)
|
||||
}
|
||||
// Why: one adapter's ignorance is not another's absence proof — only a unanimous
|
||||
// proven-absence across every generation may read as false.
|
||||
const answers = this.allAdapters().map((adapter) => adapter.hasPty(id))
|
||||
return answers.includes(true) ? true : answers.includes(null) ? null : false
|
||||
return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id))
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
@@ -317,19 +314,14 @@ export class DaemonPtyRouter implements IPtyProvider {
|
||||
}
|
||||
|
||||
private adapterForInspection(sessionId: string): DaemonPtyAdapter {
|
||||
const owner =
|
||||
const adapter =
|
||||
this.sessionAdapters.get(sessionId) ??
|
||||
this.allAdapters().find((candidate) => candidate.hasPty(sessionId) === true)
|
||||
if (owner) {
|
||||
this.sessionAdapters.set(sessionId, owner)
|
||||
return owner
|
||||
}
|
||||
// Why: only unanimous proven absence is gone; an adapter that cannot answer routes
|
||||
// to current without memoizing a route we never established.
|
||||
if (this.hasPty(sessionId) === false) {
|
||||
this.allAdapters().find((candidate) => candidate.hasPty(sessionId))
|
||||
if (!adapter) {
|
||||
throw new Error('terminal_gone')
|
||||
}
|
||||
return this.current
|
||||
this.sessionAdapters.set(sessionId, adapter)
|
||||
return adapter
|
||||
}
|
||||
|
||||
private allAdapters(): DaemonPtyAdapter[] {
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
/**
|
||||
* The evidence module decides whether a daemon still hosts user terminals by looking at its
|
||||
* process tree, and must discount the PTYs the daemon opens for itself. That exclusion list
|
||||
* is only safe while it is complete — a self-spawned PTY nobody excluded reads as user work
|
||||
* and holds a daemon that owns nothing, which is how the list grew a reviewer at a time.
|
||||
*
|
||||
* So pin the input instead of the list: every PTY the daemon opens *directly*, enumerated
|
||||
* from the source. A new spawn site fails this test until someone decides which side it
|
||||
* belongs on.
|
||||
*
|
||||
* Scope, stated so the next reader does not over-trust it: this sees node-pty calls in this
|
||||
* directory only. The daemon can also open a PTY through a helper binary — the macOS login
|
||||
* session probe shells out to `expect`, whose own `spawn` forkpty's a `login` wrapper that
|
||||
* surfaces as a session-leader grandchild (`macos-login-session-pty-probe.ts`). That one is
|
||||
* caught by the stranded-wrapper filter rather than by this list, and it is the shape a
|
||||
* future escape will take: indirect, and outside this directory.
|
||||
*/
|
||||
const KNOWN_DAEMON_PTY_SPAWN_SITES = [
|
||||
// The user's terminal — the thing the evidence exists to protect.
|
||||
{ file: 'pty-subprocess.ts', argv: 'wrapped.file, wrapped.args', hosted: true },
|
||||
// checkPtySpawnHealth
|
||||
{ file: 'pty-subprocess.ts', argv: "'/bin/sh', ['-c', 'exit 0']", hosted: false },
|
||||
// warmWindowsConptyOnce
|
||||
{ file: 'windows-conpty-warmup.ts', argv: "COMSPEC || 'cmd.exe', ['/c', 'exit']", hosted: false }
|
||||
]
|
||||
|
||||
describe('daemon self-spawned PTY inventory', () => {
|
||||
it('has no PTY spawn site the ownership evidence has not accounted for', () => {
|
||||
const daemonDir = join(import.meta.dirname)
|
||||
const sites = readdirSync(daemonDir)
|
||||
.filter((name) => name.endsWith('.ts') && !name.includes('.test.'))
|
||||
.flatMap((name) => {
|
||||
const source = readFileSync(join(daemonDir, name), 'utf8')
|
||||
return [...source.matchAll(/(?:pty\.spawn|spawnPty)\s*\(/g)]
|
||||
.filter((match) => !/typeof pty\.spawn/.test(source.slice(match.index - 80, match.index)))
|
||||
.map(() => name)
|
||||
})
|
||||
|
||||
expect(sites.sort()).toEqual(KNOWN_DAEMON_PTY_SPAWN_SITES.map((site) => site.file).sort())
|
||||
})
|
||||
})
|
||||
@@ -21,40 +21,6 @@ function provider(
|
||||
}
|
||||
|
||||
describe('DaemonSessionOwnerResolver', () => {
|
||||
it('reattaches a held daemon\u2019s session once the daemon stops being wedged', async () => {
|
||||
// Pins a promise the degraded notice makes to the user: "reopening a pane retries, and works
|
||||
// once it does". In held mode discovery ran over the same IPC the daemon was failing, so no
|
||||
// route was ever recorded — recovery therefore cannot come from a cached route. It has to
|
||||
// come from the next attach re-inventorying a provider whose failure cooldown has expired.
|
||||
const session = 'wt-1@@pane-1'
|
||||
let wedged = true
|
||||
const daemonInventory = vi.fn(async () => {
|
||||
if (wedged) {
|
||||
throw new Error('Hello response timed out')
|
||||
}
|
||||
return [{ id: session, cwd: '/repo' }] as PtyProcessInfo[]
|
||||
})
|
||||
const fallback = provider(async () => [])
|
||||
const daemon = provider(daemonInventory, async (opts) => ({
|
||||
id: opts.sessionId as string,
|
||||
isReattach: true
|
||||
}))
|
||||
const resolver = new DaemonSessionOwnerResolver([fallback, daemon], new Map())
|
||||
|
||||
// While wedged the session cannot be proven to belong to anyone, and the resolver refuses
|
||||
// rather than letting the fallback answer with a fresh shell.
|
||||
await expect(
|
||||
resolver.spawnAttachOnly({ sessionId: session, attachOnly: true } as never)
|
||||
).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError)
|
||||
|
||||
wedged = false
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_100)) // outlast FAILED_PROVIDER_COOLDOWN_MS
|
||||
|
||||
const result = await resolver.spawnAttachOnly({ sessionId: session, attachOnly: true } as never)
|
||||
expect(result.id).toBe(session)
|
||||
expect(daemon.spawn).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('coalesces complete multi-provider absence without dispatching an attach', async () => {
|
||||
let releaseFallback!: (processes: PtyProcessInfo[]) => void
|
||||
let releaseCurrent!: (processes: PtyProcessInfo[]) => void
|
||||
|
||||
@@ -28,16 +28,8 @@ export type DaemonPidFile = {
|
||||
spawnerExecPath?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 'degraded-new-pty-fallback' — adopted, but it cannot spawn fresh PTYs.
|
||||
* 'held' — deliberately kept without adopting it, because replacing it might end live work:
|
||||
* either it demonstrably owns terminals and cannot answer a handshake, or it could not be
|
||||
* classified at all. Both mean there is no lease to take, and none must be attempted.
|
||||
*/
|
||||
export type DaemonLaunchMode = 'degraded-new-pty-fallback' | 'held'
|
||||
|
||||
export type DaemonProcessHandle = {
|
||||
mode?: DaemonLaunchMode
|
||||
mode?: 'degraded-new-pty-fallback'
|
||||
releaseAdoptionLease?(): void
|
||||
shutdown(): Promise<void>
|
||||
}
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
DEGRADED_DAEMON_RECOVERY_RETRY_MS,
|
||||
DegradedDaemonFreshSpawnRouter
|
||||
} from './degraded-daemon-fresh-spawn-routing'
|
||||
import type { IPtyProvider, PtySpawnResult } from '../providers/types'
|
||||
import { DaemonProtocolError } from './daemon-errors'
|
||||
|
||||
function provider(id: string, spawn?: IPtyProvider['spawn']): IPtyProvider {
|
||||
return {
|
||||
spawn: spawn ?? vi.fn(async () => ({ id: `${id}-pty` }) as PtySpawnResult)
|
||||
} as unknown as IPtyProvider
|
||||
}
|
||||
|
||||
function router(opts: {
|
||||
probe?: (() => Promise<boolean>) | null
|
||||
currentSpawn?: IPtyProvider['spawn']
|
||||
}) {
|
||||
const current = provider('current', opts.currentSpawn)
|
||||
const fallback = provider('fallback')
|
||||
const sessionProviders = new Map<string, IPtyProvider>()
|
||||
return {
|
||||
current,
|
||||
fallback,
|
||||
sessionProviders,
|
||||
router: new DegradedDaemonFreshSpawnRouter(
|
||||
current,
|
||||
fallback,
|
||||
sessionProviders,
|
||||
opts.probe === undefined ? async () => true : opts.probe
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
describe('DegradedDaemonFreshSpawnRouter', () => {
|
||||
it('starts on the fallback, so a held daemon never receives a fresh spawn', () => {
|
||||
expect(router({}).router.routesToFallback).toBe(true)
|
||||
})
|
||||
|
||||
it('promotes fresh spawns back to the daemon once it answers a health check', async () => {
|
||||
const { router: r } = router({ probe: async () => true })
|
||||
|
||||
await expect(r.recover()).resolves.toBe(true)
|
||||
expect(r.routesToFallback).toBeUndefined()
|
||||
})
|
||||
|
||||
it('stays on the fallback while the daemon is still unhealthy', async () => {
|
||||
const { router: r } = router({ probe: async () => false })
|
||||
|
||||
await expect(r.recover()).resolves.toBe(false)
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
})
|
||||
|
||||
it('routes back to the fallback when a spawn fails after recovery', async () => {
|
||||
// The defect: recovery was a one-way flip on a two-way condition. A daemon that answers one
|
||||
// health check and wedges again kept every later fresh spawn pointed at it, and each one
|
||||
// costs a hello timeout plus a full launcher re-classification — per terminal, for the rest
|
||||
// of the session.
|
||||
const wedged = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Hello response timed out')
|
||||
})
|
||||
const { router: r } = router({ probe: async () => true, currentSpawn: wedged })
|
||||
|
||||
await r.recover()
|
||||
expect(r.routesToFallback).toBeUndefined()
|
||||
|
||||
await expect(r.spawn({ cwd: '/tmp' } as never)).rejects.toThrow('Hello response timed out')
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
})
|
||||
|
||||
it('does not immediately re-promote after routing back', async () => {
|
||||
// Without re-arming the cooldown the next spawn probes again straight away, and a wedged
|
||||
// daemon that still passes a cheap health check would be re-promoted into the same failure.
|
||||
const wedged = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Hello response timed out')
|
||||
})
|
||||
const { router: r } = router({ probe: async () => true, currentSpawn: wedged })
|
||||
|
||||
await r.recover()
|
||||
await expect(r.spawn({ cwd: '/tmp' } as never)).rejects.toThrow()
|
||||
|
||||
await expect(r.recover()).resolves.toBe(false)
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
|
||||
const past = vi
|
||||
.spyOn(Date, 'now')
|
||||
.mockReturnValue(Date.now() + DEGRADED_DAEMON_RECOVERY_RETRY_MS + 1)
|
||||
try {
|
||||
await expect(r.recover()).resolves.toBe(true)
|
||||
} finally {
|
||||
past.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('never lets a retry of a named session be answered by the fallback', async () => {
|
||||
// The dangerous case `!mapped` could not see: a spawn that names a session may already have
|
||||
// created it on the daemon and then lost the reply. Demoting on that failure would send the
|
||||
// retry to the fallback, which answers with a fresh local shell under the same id while the
|
||||
// agent keeps running on the daemon — the pane binds to the shell and the agent is orphaned.
|
||||
// That is the symptom this whole change exists to prevent, arriving by another door.
|
||||
const lostReply = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Request createSession timed out after 30000ms')
|
||||
})
|
||||
const {
|
||||
router: r,
|
||||
current,
|
||||
sessionProviders
|
||||
} = router({
|
||||
probe: async () => true,
|
||||
currentSpawn: lostReply
|
||||
})
|
||||
|
||||
await r.recover()
|
||||
await expect(r.spawn({ cwd: '/tmp', sessionId: 'wt-1@@pane-1' } as never)).rejects.toThrow()
|
||||
|
||||
// The identity sticks to the provider that may already own it...
|
||||
expect(sessionProviders.get('wt-1@@pane-1')).toBe(current)
|
||||
// ...and keeps routing there even though the shared route has since demoted, which is the
|
||||
// property that actually prevents the shadow: the pin outranks the route.
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
await expect(r.spawn({ cwd: '/tmp', sessionId: 'wt-1@@pane-1' } as never)).rejects.toThrow(
|
||||
'Request createSession timed out after 30000ms'
|
||||
)
|
||||
expect(lostReply).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('demotes for a production-shaped fresh spawn, which always carries an id', async () => {
|
||||
// The regression this pins: gating demotion on the ABSENCE of a sessionId made it
|
||||
// unreachable outside tests, because every production fresh spawn mints an id before
|
||||
// reaching the provider (ipc/pty.ts sets spawnOptions.sessionId). A recovered-then-wedged
|
||||
// daemon would keep every later terminal pointed at itself, each paying a hello timeout
|
||||
// plus a full launcher re-classification, and each failing anyway.
|
||||
const wedged = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Hello response timed out')
|
||||
})
|
||||
const { router: r, sessionProviders } = router({
|
||||
probe: async () => true,
|
||||
currentSpawn: wedged
|
||||
})
|
||||
|
||||
await r.recover()
|
||||
expect(r.routesToFallback).toBeUndefined()
|
||||
|
||||
// Exactly what ipc/pty.ts sends for a new terminal: a minted id, and no attachOnly.
|
||||
await expect(r.spawn({ cwd: '/tmp', sessionId: 'wt-1@@pane-9' } as never)).rejects.toThrow(
|
||||
'Hello response timed out'
|
||||
)
|
||||
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
// And correctly does NOT pin: a hello that never completed cannot have created a session,
|
||||
// so there is nothing on the daemon for a retry to collide with. Pinning here would strand
|
||||
// later attempts on a host holding nothing of theirs.
|
||||
expect(sessionProviders.has('wt-1@@pane-9')).toBe(false)
|
||||
})
|
||||
|
||||
it('does not demote for an attach that names a session', async () => {
|
||||
// An attachOnly spawn is not a fresh terminal; its failure says nothing about whether the
|
||||
// next new terminal should go local, and attaches with an id are routed elsewhere anyway.
|
||||
const wedged = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Hello response timed out')
|
||||
})
|
||||
const { router: r } = router({ probe: async () => true, currentSpawn: wedged })
|
||||
|
||||
await r.recover()
|
||||
await expect(
|
||||
r.spawn({ cwd: '/tmp', sessionId: 'wt-1@@pane-9', attachOnly: true } as never)
|
||||
).rejects.toThrow()
|
||||
|
||||
expect(r.routesToFallback).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not demote when the failure says nothing about the daemon', async () => {
|
||||
// A spawn can fail for reasons that are the caller's, not the host's — an unusable cwd, a
|
||||
// bad profile. Degrading the whole session's persistence for one of those costs the user
|
||||
// daemon-backed terminals the daemon would have served perfectly well.
|
||||
const rejected = vi.fn(async () => {
|
||||
throw new Error('chdir failed: ENOENT /gone')
|
||||
})
|
||||
const { router: r } = router({ probe: async () => true, currentSpawn: rejected })
|
||||
|
||||
await r.recover()
|
||||
await expect(r.spawn({ cwd: '/gone' } as never)).rejects.toThrow('chdir failed')
|
||||
expect(r.routesToFallback).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not pin a session the daemon cannot have created', async () => {
|
||||
// The pin exists for a request that was sent and whose answer was lost. A failure that never
|
||||
// reached the daemon created nothing, so pinning it would strand later attempts on a host
|
||||
// that has nothing of theirs.
|
||||
const rejected = vi.fn(async () => {
|
||||
throw new Error('chdir failed: ENOENT /gone')
|
||||
})
|
||||
const { router: r, sessionProviders } = router({
|
||||
probe: async () => true,
|
||||
currentSpawn: rejected
|
||||
})
|
||||
|
||||
await r.recover()
|
||||
await expect(r.spawn({ cwd: '/gone', sessionId: 'wt-1@@pane-3' } as never)).rejects.toThrow()
|
||||
expect(sessionProviders.has('wt-1@@pane-3')).toBe(false)
|
||||
})
|
||||
|
||||
it('still demotes on an anonymous spawn, which cannot be shadowed', async () => {
|
||||
// The case demotion exists for: no session identity, so there is nothing a fallback answer
|
||||
// could shadow, and paying a hello timeout plus a re-classification per terminal is pure loss.
|
||||
const wedged = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Hello response timed out')
|
||||
})
|
||||
const { router: r } = router({ probe: async () => true, currentSpawn: wedged })
|
||||
|
||||
await r.recover()
|
||||
await expect(r.spawn({ cwd: '/tmp' } as never)).rejects.toThrow()
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps a mapped session on its owner while sparing the next terminal', async () => {
|
||||
// A mapped id names the provider that actually owns that pty, so it must keep routing there
|
||||
// however the shared route moves. But its failure is still evidence the daemon is failing,
|
||||
// and the next *fresh* terminal is a different session that cannot be shadowed by this one —
|
||||
// so it should not have to discover the same timeout for itself.
|
||||
const wedged = vi.fn(async () => {
|
||||
throw new DaemonProtocolError('Hello response timed out')
|
||||
})
|
||||
const {
|
||||
router: r,
|
||||
current,
|
||||
sessionProviders
|
||||
} = router({
|
||||
probe: async () => true,
|
||||
currentSpawn: wedged
|
||||
})
|
||||
sessionProviders.set('session-1', current)
|
||||
// Promote first, or the assertion below passes on the constructor's default and proves nothing.
|
||||
await r.recover()
|
||||
expect(r.routesToFallback).toBeUndefined()
|
||||
|
||||
await expect(r.spawn({ cwd: '/tmp', sessionId: 'session-1' } as never)).rejects.toThrow()
|
||||
expect(sessionProviders.get('session-1')).toBe(current)
|
||||
expect(r.routesToFallback).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -1,33 +1,4 @@
|
||||
import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../providers/types'
|
||||
import { isDaemonGoneError } from './daemon-pty-adapter'
|
||||
import { DaemonProtocolError } from './daemon-errors'
|
||||
|
||||
/** client.ts rejects a sent request with this shape once its budget expires. */
|
||||
const REQUEST_TIMED_OUT = /timed out after \d+ms/
|
||||
|
||||
/**
|
||||
* Only a daemon that looks unreachable should cost the next terminal its persistence. A spawn
|
||||
* can fail for reasons that say nothing about the daemon's health — an unusable cwd, a bad
|
||||
* profile — and demoting on those degrades a session the daemon would have served fine.
|
||||
*/
|
||||
function daemonLooksUnreachable(error: unknown): boolean {
|
||||
return (
|
||||
isDaemonGoneError(error) ||
|
||||
(error instanceof DaemonProtocolError && REQUEST_TIMED_OUT.test(error.message))
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Only a request that was actually sent can hide a session the daemon created before the answer
|
||||
* was lost. A failure that never reached it cannot have created anything, so pinning that id
|
||||
* would strand later attempts on a daemon that has nothing of theirs.
|
||||
*/
|
||||
function mayHaveCreatedTheSession(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof DaemonProtocolError &&
|
||||
(error.message === 'Connection lost' || REQUEST_TIMED_OUT.test(error.message))
|
||||
)
|
||||
}
|
||||
|
||||
export const DEGRADED_DAEMON_RECOVERY_RETRY_MS = 30_000
|
||||
|
||||
@@ -97,40 +68,7 @@ export class DegradedDaemonFreshSpawnRouter {
|
||||
async spawn(opts: PtySpawnOptions): Promise<PtySpawnResult> {
|
||||
const mapped = opts.sessionId ? this.sessionProviders.get(opts.sessionId) : undefined
|
||||
const target = mapped ?? this.target
|
||||
let result: PtySpawnResult
|
||||
try {
|
||||
result = await target.spawn(opts)
|
||||
} catch (error) {
|
||||
// Why route back: recovery was a one-way flip on a two-way condition. A daemon that
|
||||
// answers one health check and wedges again kept every later spawn pointed at it, and a
|
||||
// spawn there costs a hello timeout plus a full launcher re-classification — per terminal,
|
||||
// for the rest of the session. Sending the next one to the fallback costs a terminal
|
||||
// without daemon persistence instead, and the next probe can promote it back.
|
||||
if (target === this.current) {
|
||||
// Two independent things, and conflating them cost a fix each way. Pinning protects
|
||||
// THIS id: the spawn may already have created it on the daemon and lost the reply, so
|
||||
// letting a retry reach the fallback would answer with a local shell under the same id
|
||||
// while the original keeps running. Demoting protects the NEXT terminal, which is a
|
||||
// different session entirely and cannot be shadowed by this one.
|
||||
if (opts.sessionId && mayHaveCreatedTheSession(error)) {
|
||||
this.sessionProviders.set(opts.sessionId, target)
|
||||
}
|
||||
// Why not `!opts.sessionId`: every production fresh spawn mints an id before it gets
|
||||
// here (ipc/pty.ts assigns spawnOptions.sessionId), so keying the demotion off its
|
||||
// absence made the demotion unreachable outside tests — and left every later terminal
|
||||
// paying a hello timeout plus a full re-classification against a daemon already known
|
||||
// to be failing. `attachOnly` is the real discriminator: an attach that names a session
|
||||
// never reaches this router at all.
|
||||
if (opts.attachOnly !== true && daemonLooksUnreachable(error)) {
|
||||
this.target = this.fallback
|
||||
this.retryAfterMs = Date.now() + DEGRADED_DAEMON_RECOVERY_RETRY_MS
|
||||
console.warn(
|
||||
'[daemon] Fresh terminals routed back to the local provider: the daemon failed a spawn after recovering'
|
||||
)
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const result = await target.spawn(opts)
|
||||
if (!result.exitedBeforeSpawnReply) {
|
||||
this.sessionProviders.set(result.id, target)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import type { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../providers/types'
|
||||
import type { PtyProcessInspection } from '../providers/pty-process-inspection'
|
||||
import { SessionNotFoundError, TerminalSessionOwnerUnverifiedError } from './daemon-errors'
|
||||
import { isSshPtyNotFoundError } from '../providers/ssh-pty-errors'
|
||||
|
||||
type ProviderMock = IPtyProvider & {
|
||||
probePtyLiveness: (id: string) => Promise<boolean | null>
|
||||
@@ -472,22 +471,6 @@ describe('DegradedDaemonPtyProvider', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// Why: while degraded, a provider that cannot answer must not let inspection
|
||||
// manufacture terminal_gone — that verdict retires a pane that may still be live.
|
||||
it('answers unknown, and refuses terminal_gone, when no provider can answer', async () => {
|
||||
const current = createDaemonAdapter('daemon')
|
||||
const fallback = createProvider('fallback')
|
||||
current.hasPty = vi.fn(() => null)
|
||||
fallback.hasPty = vi.fn(() => null)
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback })
|
||||
|
||||
expect(provider.hasPty('unmapped-session')).toBe(null)
|
||||
await expect(provider.inspectProcess('unmapped-session')).resolves.toEqual({
|
||||
foregroundProcess: null,
|
||||
hasChildProcesses: false
|
||||
})
|
||||
})
|
||||
|
||||
it('caches a provider discovered by hasPty before routing later operations', () => {
|
||||
const current = createDaemonAdapter('daemon', ['daemon-session'])
|
||||
const fallback = createProvider('fallback')
|
||||
@@ -664,27 +647,6 @@ describe('DegradedDaemonPtyProvider', () => {
|
||||
expect(provider.hasPty('legacy-session')).toBe(true)
|
||||
})
|
||||
|
||||
it('routes every session operation for a mapped daemon session to that adapter', async () => {
|
||||
const current = createDaemonAdapter('daemon', ['daemon-session'])
|
||||
const fallback = createProvider('fallback')
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback })
|
||||
await provider.discoverDaemonSessions()
|
||||
|
||||
provider.write('daemon-session', 'ls\n')
|
||||
provider.resize('daemon-session', 120, 40)
|
||||
await provider.sendSignal('daemon-session', 'SIGINT')
|
||||
await provider.shutdown('daemon-session', {})
|
||||
|
||||
expect(current.write).toHaveBeenCalledWith('daemon-session', 'ls\n')
|
||||
expect(current.resize).toHaveBeenCalledWith('daemon-session', 120, 40)
|
||||
expect(current.sendSignal).toHaveBeenCalledWith('daemon-session', 'SIGINT')
|
||||
expect(current.shutdown).toHaveBeenCalledWith('daemon-session', {})
|
||||
expect(fallback.write).not.toHaveBeenCalled()
|
||||
expect(fallback.resize).not.toHaveBeenCalled()
|
||||
expect(fallback.sendSignal).not.toHaveBeenCalled()
|
||||
expect(fallback.shutdown).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps an exited legacy daemon poisoning listProcesses after construction', async () => {
|
||||
const current = createDaemonAdapter('daemon', ['current-session'])
|
||||
const legacy = createDaemonAdapter('legacy', ['legacy-session'])
|
||||
@@ -700,152 +662,3 @@ describe('DegradedDaemonPtyProvider', () => {
|
||||
expect(fallback.listProcesses).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
})
|
||||
|
||||
describe('DegradedDaemonPtyProvider owner gate against an unanswerable fallback', () => {
|
||||
// STA-3077 made hasPty three-valued: null now means "this provider cannot answer", where
|
||||
// before the only answers were yes and no. The owner gate asks the fallback to *prove* it owns
|
||||
// a session before letting it act, and it must read that new null as "not proven" — otherwise
|
||||
// the in-process fallback answers for a daemon-owned session, the pane closes, and the agent
|
||||
// keeps running as an orphan. Nothing else exercises null at this boundary: every other double
|
||||
// answers false, which makes `!== true` and `=== false` indistinguishable.
|
||||
it('refuses a mutating operation when the fallback cannot answer for the session', async () => {
|
||||
const current = createDaemonAdapter('daemon')
|
||||
const fallback = createProvider('fallback')
|
||||
fallback.hasPty = vi.fn(() => null)
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback })
|
||||
|
||||
await expect(provider.shutdown('wt-1@@unanswerable', {})).rejects.toBeInstanceOf(
|
||||
TerminalSessionOwnerUnverifiedError
|
||||
)
|
||||
expect(fallback.shutdown).not.toHaveBeenCalled()
|
||||
expect(current.shutdown).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('still lets the fallback act on a session it positively claims', async () => {
|
||||
const current = createDaemonAdapter('daemon')
|
||||
const fallback = createProvider('fallback')
|
||||
fallback.hasPty = vi.fn(() => true)
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback })
|
||||
|
||||
await provider.shutdown('wt-1@@local', {})
|
||||
expect(fallback.shutdown).toHaveBeenCalledWith('wt-1@@local', {})
|
||||
})
|
||||
})
|
||||
|
||||
describe('DegradedDaemonPtyProvider with a held daemon', () => {
|
||||
const HELD_SESSION = 'wt-1@@held-daemon-session'
|
||||
|
||||
/** Held launch mode never connects to the wedged daemon, so discovery maps nothing and
|
||||
* every daemon-owned id is unrouted — i.e. resolves to the in-process fallback. */
|
||||
function createHeldDaemonProvider(): {
|
||||
current: ReturnType<typeof createDaemonAdapter>
|
||||
fallback: ReturnType<typeof createProvider>
|
||||
provider: DegradedDaemonPtyProvider
|
||||
} {
|
||||
const current = createDaemonAdapter('daemon')
|
||||
const fallback = createProvider('fallback')
|
||||
return {
|
||||
current,
|
||||
fallback,
|
||||
provider: new DegradedDaemonPtyProvider({ current, legacy: [], fallback })
|
||||
}
|
||||
}
|
||||
|
||||
it('rejects shutdown for a held daemon session instead of reporting a silent success', async () => {
|
||||
const { current, fallback, provider } = createHeldDaemonProvider()
|
||||
await provider.discoverDaemonSessions()
|
||||
|
||||
// Why: the fallback's shutdown resolves for ids it never had, so the pane would close
|
||||
// while the daemon's agent keeps running as an orphan.
|
||||
await expect(provider.shutdown(HELD_SESSION, {})).rejects.toBeInstanceOf(
|
||||
TerminalSessionOwnerUnverifiedError
|
||||
)
|
||||
expect(fallback.shutdown).not.toHaveBeenCalled()
|
||||
expect(current.shutdown).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not let the kill path mistake an unreachable owner for an already-gone pty', async () => {
|
||||
const { provider } = createHeldDaemonProvider()
|
||||
await provider.discoverDaemonSessions()
|
||||
// Mirrors pty:kill's isPtyAlreadyGoneError (src/main/ipc/pty.ts), which is not exported:
|
||||
// any error matching it is swallowed into a synthesized pty:exit and reported as success —
|
||||
// exactly the orphan-hiding lie this routing exists to prevent. Renaming the thrown error
|
||||
// back into that shape would silently reintroduce it.
|
||||
const looksAlreadyGoneToPtyKill = (error: unknown): boolean =>
|
||||
isSshPtyNotFoundError(error) ||
|
||||
/Session not found/i.test(error instanceof Error ? error.message : String(error))
|
||||
|
||||
const error = await provider.shutdown(HELD_SESSION, {}).catch((err: unknown) => err)
|
||||
|
||||
expect(error).toBeInstanceOf(TerminalSessionOwnerUnverifiedError)
|
||||
expect(looksAlreadyGoneToPtyKill(error)).toBe(false)
|
||||
})
|
||||
|
||||
it('throws on write and resize for a held daemon session instead of swallowing input', async () => {
|
||||
const { fallback, provider } = createHeldDaemonProvider()
|
||||
await provider.discoverDaemonSessions()
|
||||
|
||||
// Why: the fallback's write/resize are `ptyProcesses.get(id)?.…` — typing would vanish.
|
||||
expect(() => provider.write(HELD_SESSION, 'ls\n')).toThrow(TerminalSessionOwnerUnverifiedError)
|
||||
expect(() => provider.resize(HELD_SESSION, 120, 40)).toThrow(
|
||||
TerminalSessionOwnerUnverifiedError
|
||||
)
|
||||
expect(fallback.write).not.toHaveBeenCalled()
|
||||
expect(fallback.resize).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects sendSignal for a held daemon session', async () => {
|
||||
const { fallback, provider } = createHeldDaemonProvider()
|
||||
await provider.discoverDaemonSessions()
|
||||
|
||||
await expect(provider.sendSignal(HELD_SESSION, 'SIGINT')).rejects.toBeInstanceOf(
|
||||
TerminalSessionOwnerUnverifiedError
|
||||
)
|
||||
expect(fallback.sendSignal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps refusing attach for a held daemon session', async () => {
|
||||
const { fallback, provider } = createHeldDaemonProvider()
|
||||
await provider.discoverDaemonSessions()
|
||||
|
||||
await expect(provider.attach(HELD_SESSION)).rejects.toBeInstanceOf(SessionNotFoundError)
|
||||
expect(fallback.attach).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('still routes every operation for a locally spawned session the fallback owns', async () => {
|
||||
const { current, fallback, provider } = createHeldDaemonProvider()
|
||||
await provider.discoverDaemonSessions()
|
||||
const fresh = await provider.spawn({ cols: 80, rows: 24 })
|
||||
|
||||
provider.write(fresh.id, 'echo hi\n')
|
||||
provider.resize(fresh.id, 100, 30)
|
||||
await expect(provider.sendSignal(fresh.id, 'SIGINT')).resolves.toBeUndefined()
|
||||
await expect(provider.shutdown(fresh.id, {})).resolves.toBeUndefined()
|
||||
|
||||
expect(fallback.write).toHaveBeenCalledWith(fresh.id, 'echo hi\n')
|
||||
expect(fallback.resize).toHaveBeenCalledWith(fresh.id, 100, 30)
|
||||
expect(fallback.sendSignal).toHaveBeenCalledWith(fresh.id, 'SIGINT')
|
||||
expect(fallback.shutdown).toHaveBeenCalledWith(fresh.id, {})
|
||||
expect(current.write).not.toHaveBeenCalled()
|
||||
expect(current.shutdown).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
// A memoized route outlives the session it was established for: listProcesses
|
||||
// drops ids missing from an authoritative inventory without an exit fanout. So a
|
||||
// mapped owner that cannot answer must stay unknown — coercing it to a liveness
|
||||
// proof is worse than the absence it replaced, because callers skip the real probe.
|
||||
it('keeps a mapped owner that cannot answer unknown, and still probes', async () => {
|
||||
const current = createDaemonAdapter('current', ['s1'])
|
||||
const fallback = createProvider('fallback')
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [], fallback })
|
||||
|
||||
expect(provider.hasPty('s1')).toBe(true)
|
||||
|
||||
current.hasPty = vi.fn(() => null)
|
||||
expect(provider.hasPty('s1')).toBeNull()
|
||||
|
||||
current.probePtyLiveness = vi.fn(async () => null)
|
||||
await provider.probePtyLiveness('s1')
|
||||
expect(current.probePtyLiveness).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -14,7 +14,6 @@ import type {
|
||||
import {
|
||||
adoptOwningProvider,
|
||||
attachDaemonOwnedSession,
|
||||
ownerForDaemonOwnedOperation,
|
||||
findDaemonAdapter,
|
||||
listProviderSessionIds
|
||||
} from './degraded-daemon-session-routing'
|
||||
@@ -90,30 +89,14 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
attach = (id: string): ReturnType<IPtyProvider['attach']> =>
|
||||
attachDaemonOwnedSession(this.providerFor(id), this.fallback, id)
|
||||
|
||||
/** Routing for anything that changes or feeds a session; see ownerForDaemonOwnedOperation. */
|
||||
private ownerFor(id: string): IPtyProvider {
|
||||
return ownerForDaemonOwnedOperation(this.providerFor(id), this.fallback, id)
|
||||
}
|
||||
|
||||
hasPty(id: string): boolean | null {
|
||||
hasPty(id: string): boolean {
|
||||
const mapped = this.sessionProviders.get(id)
|
||||
if (mapped) {
|
||||
// Why not `?? true`: a route outlives the session it was memoized for, so an
|
||||
// owner that cannot answer must stay unknown rather than become a liveness proof.
|
||||
return mapped.hasPty ? mapped.hasPty(id) : true
|
||||
}
|
||||
if (this.findProviderForExistingSession(id)) {
|
||||
return true
|
||||
}
|
||||
// Why: one provider that cannot answer makes absence unknown; only unanimous proof is false.
|
||||
return this.allProviders().every((provider) => provider.hasPty?.(id) === false) ? false : null
|
||||
return mapped ? (mapped.hasPty?.(id) ?? true) : this.findProviderForExistingSession(id) !== null
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
const mapped = this.sessionProviders.get(id)
|
||||
// Why only an explicit true short-circuits: an owner that cannot answer must
|
||||
// fall through to the real probe instead of skipping it on a fabricated proof.
|
||||
if (mapped && (mapped.hasPty ? mapped.hasPty(id) === true : true)) {
|
||||
if (mapped && (mapped.hasPty?.(id) ?? true)) {
|
||||
return true
|
||||
}
|
||||
return await this.ownerRecovery.probe(id)
|
||||
@@ -126,11 +109,11 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
)?.providesAgentSessionOwnerListings?.(ptyId) === true
|
||||
|
||||
write(id: string, data: string): void {
|
||||
this.ownerFor(id).write(id, data)
|
||||
this.providerFor(id).write(id, data)
|
||||
}
|
||||
|
||||
resize(id: string, cols: number, rows: number): void {
|
||||
this.ownerFor(id).resize(id, cols, rows)
|
||||
this.providerFor(id).resize(id, cols, rows)
|
||||
}
|
||||
|
||||
pauseProducer(id: string): void {
|
||||
@@ -149,14 +132,14 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
id: string,
|
||||
opts: { immediate?: boolean; keepHistory?: boolean; deadlineMs?: number }
|
||||
): Promise<void> {
|
||||
await this.ownerFor(id).shutdown(id, opts)
|
||||
await this.providerFor(id).shutdown(id, opts)
|
||||
if (!opts.keepHistory) {
|
||||
this.sessionProviders.delete(id)
|
||||
}
|
||||
}
|
||||
|
||||
async sendSignal(id: string, signal: string): Promise<void> {
|
||||
await this.ownerFor(id).sendSignal(id, signal)
|
||||
await this.providerFor(id).sendSignal(id, signal)
|
||||
}
|
||||
|
||||
async getCwd(id: string): Promise<string> {
|
||||
@@ -197,9 +180,9 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
return this.providerFor(id).getForegroundProcess(id)
|
||||
}
|
||||
inspectProcess(id: string) {
|
||||
return this.hasPty(id) === false
|
||||
? Promise.reject(new Error('terminal_gone'))
|
||||
: inspectPtyProviderProcess(this.providerFor(id), id)
|
||||
return this.hasPty(id)
|
||||
? inspectPtyProviderProcess(this.providerFor(id), id)
|
||||
: Promise.reject(new Error('terminal_gone'))
|
||||
}
|
||||
async confirmForegroundProcess(id: string): Promise<string | null> {
|
||||
return this.providerFor(id).confirmForegroundProcess?.(id) ?? null
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { IPtyProvider } from '../providers/types'
|
||||
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
import { SessionNotFoundError, TerminalSessionOwnerUnverifiedError } from './daemon-errors'
|
||||
import { SessionNotFoundError } from './daemon-errors'
|
||||
|
||||
export function listProviderSessionIds(
|
||||
sessionProviders: ReadonlyMap<string, IPtyProvider>,
|
||||
@@ -26,29 +26,6 @@ export async function attachDaemonOwnedSession(
|
||||
return await owner.attach(sessionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Session operations that must never be answered by the in-process fallback on another
|
||||
* provider's behalf. An unknown id resolves to the fallback, whose shutdown returns
|
||||
* silently and whose write/resize are no-ops — so a daemon-owned session reads as closed
|
||||
* while its agent keeps running, and typing into it disappears. Route there only when the
|
||||
* fallback genuinely owns the pty; otherwise say the session cannot be reached.
|
||||
*
|
||||
* Why not SessionNotFoundError: pty:kill treats "Session not found" as proof the pty is
|
||||
* already gone and synthesizes an exit, which is the same lie by another route. This one
|
||||
* means "still there, we just cannot reach its host", so the kill is reported as failed and
|
||||
* ownership is kept for a retry.
|
||||
*/
|
||||
export function ownerForDaemonOwnedOperation(
|
||||
owner: IPtyProvider,
|
||||
fallback: IPtyProvider,
|
||||
sessionId: string
|
||||
): IPtyProvider {
|
||||
if (owner === fallback && fallback.hasPty?.(sessionId) !== true) {
|
||||
throw new TerminalSessionOwnerUnverifiedError(sessionId)
|
||||
}
|
||||
return owner
|
||||
}
|
||||
|
||||
/** Probes providers for an id absent from the routing map and adopts the
|
||||
* first proven owner into the map. */
|
||||
export function adoptOwningProvider(
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
/**
|
||||
* A mutating operation must not reach a PTY that has been superseded for its
|
||||
* pane. Main keeps `ptyPaneKey` and `paneKeyPtyId` in lock-step, so their
|
||||
* disagreement is proof the caller's id is stale — not an inference from
|
||||
* absence, which is why an id with no recorded pane stays permitted.
|
||||
*
|
||||
* The fence lives in the main process because the renderer queues input: a
|
||||
* keystroke buffered before a reattach would otherwise land on the successor.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
/** Mirrors the maps and predicate in pty.ts. Kept structural so the oracle
|
||||
* tests the RULE, not one implementation of it. */
|
||||
function makeFence() {
|
||||
const ptyPaneKey = new Map<string, string>()
|
||||
const paneKeyPtyId = new Map<string, string>()
|
||||
const bind = (paneKey: string, ptyId: string): void => {
|
||||
ptyPaneKey.set(ptyId, paneKey)
|
||||
paneKeyPtyId.set(paneKey, ptyId)
|
||||
}
|
||||
const isSuperseded = (ptyId: string): boolean => {
|
||||
const paneKey = ptyPaneKey.get(ptyId)
|
||||
if (paneKey === undefined) {
|
||||
return false
|
||||
}
|
||||
const currentPtyId = paneKeyPtyId.get(paneKey)
|
||||
return currentPtyId !== undefined && currentPtyId !== ptyId
|
||||
}
|
||||
return { bind, isSuperseded, ptyPaneKey, paneKeyPtyId }
|
||||
}
|
||||
|
||||
const PANE = 'tab-1:3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90'
|
||||
|
||||
describe('superseded PTY operation fence', () => {
|
||||
it('refuses an operation whose pane has since bound a different PTY', () => {
|
||||
const fence = makeFence()
|
||||
fence.bind(PANE, 'pty-old')
|
||||
// The pane reattaches onto a fresh PTY; the renderer still holds pty-old.
|
||||
fence.bind(PANE, 'pty-new')
|
||||
|
||||
expect(fence.isSuperseded('pty-old')).toBe(true)
|
||||
expect(fence.isSuperseded('pty-new')).toBe(false)
|
||||
})
|
||||
|
||||
it('permits the current PTY for its own pane', () => {
|
||||
const fence = makeFence()
|
||||
fence.bind(PANE, 'pty-live')
|
||||
|
||||
expect(fence.isSuperseded('pty-live')).toBe(false)
|
||||
})
|
||||
|
||||
// Unknown is not stale. Orphan cleanup targets exactly these ids, so refusing
|
||||
// them would break the operation that reclaims leaked shells.
|
||||
it('permits a PTY that owns no pane', () => {
|
||||
const fence = makeFence()
|
||||
|
||||
expect(fence.isSuperseded('pty-orphan')).toBe(false)
|
||||
})
|
||||
|
||||
it('permits a PTY whose pane record was dropped', () => {
|
||||
const fence = makeFence()
|
||||
fence.bind(PANE, 'pty-live')
|
||||
fence.paneKeyPtyId.delete(PANE)
|
||||
|
||||
expect(fence.isSuperseded('pty-live')).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps sibling panes independent', () => {
|
||||
const fence = makeFence()
|
||||
const other = 'tab-1:8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d'
|
||||
fence.bind(PANE, 'pty-a')
|
||||
fence.bind(other, 'pty-b')
|
||||
fence.bind(PANE, 'pty-a2')
|
||||
|
||||
expect(fence.isSuperseded('pty-a')).toBe(true)
|
||||
expect(fence.isSuperseded('pty-b')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('the fence is wired into every mutating handler', () => {
|
||||
it('combines supersession and incarnation in the write fence', async () => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const source = readFileSync('src/main/ipc/pty.ts', 'utf-8')
|
||||
const start = source.indexOf('const isCurrentPtyWrite')
|
||||
|
||||
expect(start).toBeGreaterThan(0)
|
||||
expect(source.slice(start, start + 300)).toContain('isSupersededPtyId')
|
||||
expect(source.slice(start, start + 300)).toContain('ptyIncarnationById')
|
||||
})
|
||||
|
||||
// The capability existed for months and was never called from these handlers.
|
||||
// Pin the call site, not the capability — that is the failure this program hit.
|
||||
it.each(['pty:write', 'pty:writeAccepted', 'pty:resize', 'pty:signal'])(
|
||||
'%s consults the fence',
|
||||
async (channel) => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const source = readFileSync('src/main/ipc/pty.ts', 'utf-8')
|
||||
// Anchor on the registration, not the first mention of the name.
|
||||
const start = source.search(
|
||||
new RegExp(`ipcMain\\.(?:on|handle)\\(\\s*'${channel.replace(':', ':')}'`)
|
||||
)
|
||||
expect(start, `${channel} handler not found`).toBeGreaterThan(0)
|
||||
expect(source.slice(start, start + 700)).toContain(
|
||||
channel.startsWith('pty:write') ? 'isCurrentPtyWrite' : 'isSupersededPtyId'
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
// pty:kill is deliberately NOT fenced: a superseded PTY is orphaned, and
|
||||
// reclaiming it is exactly what the orphan-cleanup callers ask for.
|
||||
it('leaves pty:kill unfenced on purpose', async () => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const source = readFileSync('src/main/ipc/pty.ts', 'utf-8')
|
||||
const start = source.search(/ipcMain\.handle\(\s*'pty:kill'/)
|
||||
expect(start).toBeGreaterThan(0)
|
||||
expect(source.slice(start, start + 500)).not.toContain('isSupersededPtyId')
|
||||
})
|
||||
})
|
||||
+41
-386
@@ -245,7 +245,6 @@ import {
|
||||
getLocalPtyProvider,
|
||||
isCurrentPtyExit,
|
||||
restorePtyIncarnation,
|
||||
bindPaneShell,
|
||||
type PrepareCodexSessionResume
|
||||
} from './pty'
|
||||
import { __resetPersistedWindowsPathCacheForTests } from '../pty/windows-environment-path'
|
||||
@@ -5304,10 +5303,7 @@ describe('registerPtyHandlers', () => {
|
||||
paneKey?: string
|
||||
tabId?: string
|
||||
}) => ({
|
||||
id: 'ssh-pty',
|
||||
// The host attests the shell's identity at spawn; the lease has to carry it or the
|
||||
// reconnect fence has nothing to compare and attaches by pty id alone.
|
||||
incarnationId: 'inc-host-ssh-pty'
|
||||
id: 'ssh-pty'
|
||||
})
|
||||
)
|
||||
const store = {
|
||||
@@ -5393,23 +5389,18 @@ describe('registerPtyHandlers', () => {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
// Without this the field exists, loads and is read — and is never written, so the
|
||||
// reconnect fence silently permits everything while every other clause stays green.
|
||||
incarnationId: 'inc-host-ssh-pty',
|
||||
state: 'attached'
|
||||
})
|
||||
)
|
||||
// STA-3077 step P inverted this: SSH pane bindings no longer go to a
|
||||
// per-target `ssh:<target>` partition. One home (`local`) only, so the
|
||||
// hostId argument is gone — pin its absence, not its old value.
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith({
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
ptyId: 'ssh-pty',
|
||||
incarnationId: 'inc-host-ssh-pty'
|
||||
})
|
||||
expect(store.persistPtyBinding.mock.calls.at(-1)).toHaveLength(1)
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith(
|
||||
{
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
ptyId: 'ssh-pty'
|
||||
},
|
||||
'ssh:ssh-1'
|
||||
)
|
||||
|
||||
store.upsertSshRemotePtyLease.mockClear()
|
||||
store.persistPtyBinding.mockClear()
|
||||
@@ -6843,38 +6834,6 @@ describe('registerPtyHandlers', () => {
|
||||
expect(fallbackShutdown).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Why: before the swap lands the pre-swap local provider owns no daemon id, so its
|
||||
// "not in my table" would answer an authoritative dead for every restored session.
|
||||
it('waits for the desktop startup barrier before pty:hasPty answers for a daemon id', async () => {
|
||||
const barrier = makeDeferred()
|
||||
const awaitLocalPtyProviderStartup = vi.fn(() => barrier.promise)
|
||||
registerPtyHandlers(
|
||||
mainWindow as never,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
{
|
||||
awaitLocalPtyProviderStartup
|
||||
}
|
||||
)
|
||||
|
||||
const pending = handlers.get('pty:hasPty')!(null, { id: 'daemon-session' }) as Promise<
|
||||
boolean | null
|
||||
>
|
||||
await Promise.resolve()
|
||||
expect(awaitLocalPtyProviderStartup).toHaveBeenCalledTimes(1)
|
||||
|
||||
installObservableDaemonTestProvider()
|
||||
;(getLocalPtyProvider() as { hasPty?: (id: string) => boolean | null }).hasPty = vi.fn(
|
||||
(id: string) => id === 'daemon-session'
|
||||
)
|
||||
barrier.resolve()
|
||||
|
||||
await expect(pending).resolves.toBe(true)
|
||||
})
|
||||
|
||||
it('waits for the desktop startup barrier before runtime local kills resolve the provider', async () => {
|
||||
const barrier = makeDeferred()
|
||||
const awaitLocalPtyProviderStartup = vi.fn(() => barrier.promise)
|
||||
@@ -7895,7 +7854,6 @@ describe('registerPtyHandlers', () => {
|
||||
id: opts.sessionId ?? 'daemon-pty'
|
||||
})),
|
||||
write,
|
||||
hasPty: vi.fn(() => true),
|
||||
resize: vi.fn(args.resize ?? (() => {})),
|
||||
getAppliedSize: vi.fn(args.getAppliedSize ?? (async () => args.applied)),
|
||||
kill: vi.fn(),
|
||||
@@ -8120,107 +8078,6 @@ describe('registerPtyHandlers', () => {
|
||||
expect(write).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not forward queued host input after the pane rebinds', async () => {
|
||||
const write = setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 } })
|
||||
let resolveClaim: (claimed: boolean) => void = () => {}
|
||||
const claimRemoteDesktopHost = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveClaim = resolve
|
||||
})
|
||||
)
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
createPreAllocatedTerminalHandle: vi.fn(() => null),
|
||||
registerPty: vi.fn(),
|
||||
getDriver: vi.fn(() => ({ kind: 'idle' })),
|
||||
claimRemoteDesktopHost,
|
||||
onPtySpawned: vi.fn(),
|
||||
onPtyExit: vi.fn(),
|
||||
onPtyData: vi.fn()
|
||||
}
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
const spawn = await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24, env: {} })
|
||||
const id = (spawn as { id: string }).id
|
||||
const leafId = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90'
|
||||
const bindingStore = {
|
||||
persistPtyBinding: vi.fn(() => true),
|
||||
getWorkspaceSession: vi.fn(() => getDefaultWorkspaceSession())
|
||||
}
|
||||
bindPaneShell({
|
||||
store: bindingStore,
|
||||
worktreeId: 'repo-1:wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
ptyId: id
|
||||
})
|
||||
const claim = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:claimViewport')
|
||||
const writeEvent = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:write')
|
||||
|
||||
claim?.[1](mainWindowIpcEvent, { id, cols: 125, rows: 48 })
|
||||
writeEvent?.[1](mainWindowIpcEvent, { id, data: 'queued' })
|
||||
const accepted = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, {
|
||||
id,
|
||||
data: 'accepted'
|
||||
})
|
||||
bindPaneShell({
|
||||
store: bindingStore,
|
||||
worktreeId: 'repo-1:wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId,
|
||||
ptyId: 'pty-successor'
|
||||
})
|
||||
resolveClaim(true)
|
||||
|
||||
await expect(accepted).resolves.toBe(false)
|
||||
await Promise.resolve()
|
||||
expect(write).not.toHaveBeenCalled()
|
||||
clearProviderPtyState(id)
|
||||
clearProviderPtyState('pty-successor')
|
||||
})
|
||||
|
||||
it('does not forward queued host input after the PTY incarnation changes', async () => {
|
||||
const write = setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 } })
|
||||
let resolveClaim: (claimed: boolean) => void = () => {}
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
createPreAllocatedTerminalHandle: vi.fn(() => null),
|
||||
registerPty: vi.fn(),
|
||||
getDriver: vi.fn(() => ({ kind: 'idle' })),
|
||||
claimRemoteDesktopHost: vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveClaim = resolve
|
||||
})
|
||||
),
|
||||
onPtySpawned: vi.fn(),
|
||||
onPtyExit: vi.fn(),
|
||||
onPtyData: vi.fn()
|
||||
}
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
const spawn = await handlers.get('pty:spawn')!(null, { cols: 80, rows: 24, env: {} })
|
||||
const id = (spawn as { id: string }).id
|
||||
restorePtyIncarnation(id, 'incarnation-old')
|
||||
const claim = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:claimViewport')
|
||||
const writeEvent = onMock.mock.calls.find((entry: unknown[]) => entry[0] === 'pty:write')
|
||||
|
||||
claim?.[1](mainWindowIpcEvent, { id, cols: 125, rows: 48 })
|
||||
writeEvent?.[1](mainWindowIpcEvent, { id, data: 'queued' })
|
||||
const accepted = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, {
|
||||
id,
|
||||
data: 'accepted'
|
||||
})
|
||||
restorePtyIncarnation(id, 'incarnation-new')
|
||||
resolveClaim(true)
|
||||
|
||||
await expect(accepted).resolves.toBe(false)
|
||||
await Promise.resolve()
|
||||
expect(write).not.toHaveBeenCalled()
|
||||
clearProviderPtyState(id)
|
||||
})
|
||||
|
||||
it('does not populate the remote reclaim cache when only a phone drives', async () => {
|
||||
const resizeSpy = vi.fn()
|
||||
setupProviderWithAppliedSize({ applied: { cols: 80, rows: 24 }, resize: resizeSpy })
|
||||
@@ -8822,17 +8679,12 @@ describe('registerPtyHandlers', () => {
|
||||
}
|
||||
}): Promise<{ id: string }>
|
||||
}
|
||||
let emitExit: ((event: { exitCode: number; signal: number }) => void) | undefined
|
||||
const kill = vi.fn(() => emitExit?.({ exitCode: 0, signal: 0 }))
|
||||
const proc = {
|
||||
onData: vi.fn(),
|
||||
onExit: vi.fn((callback) => {
|
||||
emitExit = callback
|
||||
return makeDisposable()
|
||||
}),
|
||||
onExit: vi.fn(),
|
||||
write: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
kill,
|
||||
kill: vi.fn(),
|
||||
process: 'zsh',
|
||||
pid: 12345
|
||||
}
|
||||
@@ -8887,7 +8739,7 @@ describe('registerPtyHandlers', () => {
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ expectedSourceBinding })
|
||||
)
|
||||
expect(kill).toHaveBeenCalledOnce()
|
||||
expect(proc.kill).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('reports lower-owner commit before rejecting an early-exited runtime incarnation', async () => {
|
||||
@@ -9801,9 +9653,6 @@ describe('registerPtyHandlers', () => {
|
||||
)
|
||||
expect(runtime.noteTerminalSpawnCommand).not.toHaveBeenCalled()
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledOnce()
|
||||
// STA-3077 step P inverted this: the reattach writer no longer passes a
|
||||
// hostId slot at all (it used to pass `undefined` here for the local case),
|
||||
// so the call is now strictly single-argument.
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
worktreeId,
|
||||
@@ -9815,10 +9664,9 @@ describe('registerPtyHandlers', () => {
|
||||
ptyId: 'pty-persisted-owner',
|
||||
incarnationId: 'inc-stale-owner'
|
||||
}
|
||||
})
|
||||
}),
|
||||
undefined
|
||||
)
|
||||
expect(store.persistPtyBinding.mock.calls[0]!).toHaveLength(1)
|
||||
expect(getPtyIdForPaneKey(paneKey)).toBe('pty-persisted-owner')
|
||||
expect(
|
||||
mainWindow.webContents.send.mock.calls.filter(([channel]) => channel === 'pty:spawned')
|
||||
).toHaveLength(1)
|
||||
@@ -10517,139 +10365,9 @@ describe('registerPtyHandlers', () => {
|
||||
expect(runtime.onPtyExit).toHaveBeenCalledWith('pty-already-retired-owner', 0, undefined)
|
||||
})
|
||||
|
||||
// Sibling of the clause above, and the reason that one is driven by a proof. Retiring an owner
|
||||
// authorizes a replacement carrying the pane's agent resume payload, so a bare not-found must
|
||||
// not do it: a replaced relay answers exactly that for shells its predecessor still runs.
|
||||
it('does not retire an SSH owner, nor respawn it, on a bare not-found', async () => {
|
||||
const connectionId = 'ssh-unproven-stable-pane'
|
||||
const tabId = 'tab-unproven-ssh-owner'
|
||||
const leafId = '35353535-3535-4535-8535-353535353535'
|
||||
const paneKey = makePaneKey(tabId, leafId)
|
||||
const worktreeId = 'repo-ssh::/remote/unproven-stable-pane'
|
||||
const livePtyId = `ssh:${connectionId}@@live-relay-pty`
|
||||
const freshPtyId = `ssh:${connectionId}@@fresh-relay-pty`
|
||||
const remoteSpawn = vi.fn(async (options: { attachOnly?: boolean; command?: string }) => {
|
||||
if (options.attachOnly) {
|
||||
// The reattach mints this only after verifying the relay's observed exit names this shell.
|
||||
// A bare not-found reaches here too and must NOT retire the owner — pinned by the sibling
|
||||
// clause below, because retirement authorizes a replacement carrying the resume payload.
|
||||
throw new Error('PTY "live-relay-pty" not found')
|
||||
}
|
||||
return { id: freshPtyId, incarnationId: 'inc-fresh-ssh-owner' }
|
||||
})
|
||||
registerSshPtyProvider(connectionId, {
|
||||
spawn: remoteSpawn,
|
||||
write: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
shutdown: vi.fn(),
|
||||
sendSignal: vi.fn(),
|
||||
getCwd: vi.fn(),
|
||||
getInitialCwd: vi.fn(),
|
||||
clearBuffer: vi.fn(),
|
||||
acknowledgeDataEvent: vi.fn(),
|
||||
onData: vi.fn(() => () => {}),
|
||||
onReplay: vi.fn(() => () => {}),
|
||||
onExit: vi.fn(() => () => {}),
|
||||
listProcesses: vi.fn(),
|
||||
hasChildProcesses: vi.fn(),
|
||||
getForegroundProcess: vi.fn(),
|
||||
serialize: vi.fn(),
|
||||
revive: vi.fn(),
|
||||
getDefaultShell: vi.fn(),
|
||||
getProfiles: vi.fn()
|
||||
} as never)
|
||||
let session = {
|
||||
tabsByWorktree: {
|
||||
[worktreeId]: [{ id: tabId, worktreeId, ptyId: livePtyId }]
|
||||
},
|
||||
terminalLayoutsByTabId: {
|
||||
[tabId]: {
|
||||
root: { type: 'leaf' as const, leafId },
|
||||
activeLeafId: leafId,
|
||||
expandedLeafId: null,
|
||||
ptyIdsByLeafId: { [leafId]: livePtyId }
|
||||
}
|
||||
},
|
||||
terminalPtyIncarnationsByPaneKey: { [paneKey]: 'inc-live-ssh-owner' }
|
||||
}
|
||||
const store = {
|
||||
// STA-3077 step P inverted these: the reader/retirer used to target the
|
||||
// per-target `ssh:<connectionId>` partition while the renderer published
|
||||
// pane membership to `local` — two homes, so supersession no-opped. Both
|
||||
// sides now address the single default (local) partition, i.e. no hostId.
|
||||
getWorkspaceSession: vi.fn((requestedHostId?: string) => {
|
||||
expect(requestedHostId).toBeUndefined()
|
||||
return session
|
||||
}),
|
||||
setWorkspaceSession: vi.fn((next, requestedHostId?: string) => {
|
||||
expect(requestedHostId).toBeUndefined()
|
||||
session = next
|
||||
}),
|
||||
flushOrThrow: vi.fn(),
|
||||
persistPtyBinding: vi.fn(),
|
||||
upsertSshRemotePtyLease: vi.fn(),
|
||||
removeSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLease: vi.fn()
|
||||
}
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
resolveTerminalPane: vi.fn(() => {
|
||||
throw new Error('terminal_not_found')
|
||||
}),
|
||||
createPreAllocatedTerminalHandle: vi.fn(() => 'term-fresh-ssh-owner'),
|
||||
registerPreAllocatedHandleForPty: vi.fn(),
|
||||
beginPtyRegistration: vi.fn(),
|
||||
cancelPendingPtyRegistration: vi.fn(),
|
||||
assertPtyRegistrationAllowed: vi.fn(),
|
||||
registerPty: vi.fn(),
|
||||
noteTerminalSpawnCommand: vi.fn(),
|
||||
seedHeadlessTerminal: vi.fn(),
|
||||
getDriver: vi.fn(() => ({ kind: 'host' })),
|
||||
onPtySpawned: vi.fn(),
|
||||
onPtyExit: vi.fn(),
|
||||
onPtyData: vi.fn()
|
||||
}
|
||||
|
||||
try {
|
||||
registerPtyHandlers(
|
||||
mainWindow as never,
|
||||
runtime as never,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
store as never
|
||||
)
|
||||
await expect(
|
||||
handlers.get('pty:spawn')!(null, {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: '/remote/unproven-stable-pane',
|
||||
command: 'codex resume a-session-that-may-still-be-running',
|
||||
connectionId,
|
||||
worktreeId,
|
||||
tabId,
|
||||
leafId,
|
||||
env: {
|
||||
ORCA_PANE_KEY: paneKey,
|
||||
ORCA_TAB_ID: tabId,
|
||||
ORCA_WORKTREE_ID: worktreeId
|
||||
}
|
||||
})
|
||||
).rejects.toThrow(/not found/i)
|
||||
|
||||
// Only the attach ran: no replacement carrying the resume payload.
|
||||
expect(remoteSpawn).toHaveBeenCalledTimes(1)
|
||||
expect(remoteSpawn.mock.calls[0]?.[0]).toMatchObject({ attachOnly: true })
|
||||
// The pane keeps its owner, so the shell stays reattachable.
|
||||
expect(store.setWorkspaceSession).not.toHaveBeenCalled()
|
||||
expect(runtime.onPtyExit).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
unregisterSshPtyProvider(connectionId)
|
||||
}
|
||||
})
|
||||
|
||||
it('retires a dead owner from the local session — the one pane-binding home — before fresh recovery', async () => {
|
||||
it('retires a dead owner from the exact SSH host session before fresh recovery', async () => {
|
||||
const connectionId = 'ssh-dead-stable-pane'
|
||||
const hostId = `ssh:${connectionId}`
|
||||
const tabId = 'tab-dead-ssh-owner'
|
||||
const leafId = '34343434-3434-4434-8434-343434343434'
|
||||
const paneKey = makePaneKey(tabId, leafId)
|
||||
@@ -10658,8 +10376,7 @@ describe('registerPtyHandlers', () => {
|
||||
const freshPtyId = `ssh:${connectionId}@@fresh-relay-pty`
|
||||
const remoteSpawn = vi.fn(async (options: { attachOnly?: boolean; command?: string }) => {
|
||||
if (options.attachOnly) {
|
||||
// Proven exit: only this error authorizes retiring the owner (bare not-found must not).
|
||||
throw new Error('SSH_SESSION_EXPIRED: dead-relay-pty')
|
||||
throw new Error('PTY "dead-relay-pty" not found')
|
||||
}
|
||||
return { id: freshPtyId, incarnationId: 'inc-fresh-ssh-owner' }
|
||||
})
|
||||
@@ -10699,16 +10416,12 @@ describe('registerPtyHandlers', () => {
|
||||
terminalPtyIncarnationsByPaneKey: { [paneKey]: 'inc-dead-ssh-owner' }
|
||||
}
|
||||
const store = {
|
||||
// STA-3077 step P inverted these: the reader/retirer used to target the
|
||||
// per-target `ssh:<connectionId>` partition while the renderer published
|
||||
// pane membership to `local` — two homes, so supersession no-opped. Both
|
||||
// sides now address the single default (local) partition, i.e. no hostId.
|
||||
getWorkspaceSession: vi.fn((requestedHostId?: string) => {
|
||||
expect(requestedHostId).toBeUndefined()
|
||||
expect(requestedHostId).toBe(hostId)
|
||||
return session
|
||||
}),
|
||||
setWorkspaceSession: vi.fn((next, requestedHostId?: string) => {
|
||||
expect(requestedHostId).toBeUndefined()
|
||||
expect(requestedHostId).toBe(hostId)
|
||||
session = next
|
||||
}),
|
||||
flushOrThrow: vi.fn(),
|
||||
@@ -10771,25 +10484,16 @@ describe('registerPtyHandlers', () => {
|
||||
expect(remoteSpawn.mock.calls[1]?.[0]).toMatchObject({
|
||||
command: 'codex resume exact-dead-ssh-provider-session'
|
||||
})
|
||||
// STA-3077 step P inverted these: the retirement write and the fresh
|
||||
// binding write both land in the single default (local) partition now, so
|
||||
// no hostId is passed. The dead owner must still actually be retired.
|
||||
expect(store.setWorkspaceSession).toHaveBeenCalledOnce()
|
||||
expect(store.setWorkspaceSession.mock.calls[0]!).toHaveLength(1)
|
||||
expect(
|
||||
store.setWorkspaceSession.mock.calls[0]![0].terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[
|
||||
leafId
|
||||
]
|
||||
).not.toBe(deadPtyId)
|
||||
expect(store.setWorkspaceSession).toHaveBeenCalledWith(expect.anything(), hostId)
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
worktreeId,
|
||||
tabId,
|
||||
leafId,
|
||||
ptyId: freshPtyId
|
||||
})
|
||||
}),
|
||||
hostId
|
||||
)
|
||||
expect(store.persistPtyBinding.mock.calls.at(-1)!).toHaveLength(1)
|
||||
} finally {
|
||||
unregisterSshPtyProvider(connectionId)
|
||||
}
|
||||
@@ -11135,15 +10839,15 @@ describe('registerPtyHandlers', () => {
|
||||
state: 'attached'
|
||||
})
|
||||
)
|
||||
// STA-3077 step P inverted this: the headless binding writer no longer
|
||||
// targets `ssh:<target>`; one home (`local`) only, so no hostId argument.
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith({
|
||||
worktreeId: 'wt-remote',
|
||||
tabId: 'tab-remote',
|
||||
leafId,
|
||||
ptyId: 'ssh:ssh-1@@relay-pty'
|
||||
})
|
||||
expect(store.persistPtyBinding.mock.calls[0]!).toHaveLength(1)
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith(
|
||||
{
|
||||
worktreeId: 'wt-remote',
|
||||
tabId: 'tab-remote',
|
||||
leafId,
|
||||
ptyId: 'ssh:ssh-1@@relay-pty'
|
||||
},
|
||||
'ssh:ssh-1'
|
||||
)
|
||||
expect(store.persistPtyBinding.mock.invocationCallOrder[0]!).toBeLessThan(
|
||||
store.upsertSshRemotePtyLease.mock.invocationCallOrder[0]!
|
||||
)
|
||||
@@ -11287,15 +10991,15 @@ describe('registerPtyHandlers', () => {
|
||||
persistHostSessionBinding: true
|
||||
})
|
||||
|
||||
// STA-3077 step P inverted this: the reattach binding refresh writes to the
|
||||
// single default (local) partition, so the `ssh:<target>` hostId is gone.
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith({
|
||||
worktreeId: 'wt-remote',
|
||||
tabId: 'tab-remote',
|
||||
leafId,
|
||||
ptyId: 'ssh:ssh-reattach-ok@@relay-pty'
|
||||
})
|
||||
expect(store.persistPtyBinding.mock.calls.at(-1)!).toHaveLength(1)
|
||||
expect(store.persistPtyBinding).toHaveBeenCalledWith(
|
||||
{
|
||||
worktreeId: 'wt-remote',
|
||||
tabId: 'tab-remote',
|
||||
leafId,
|
||||
ptyId: 'ssh:ssh-reattach-ok@@relay-pty'
|
||||
},
|
||||
'ssh:ssh-reattach-ok'
|
||||
)
|
||||
expect(store.upsertSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
targetId: 'ssh-reattach-ok',
|
||||
@@ -17970,31 +17674,6 @@ describe('registerPtyHandlers', () => {
|
||||
expect(mockProc.proc.write).toHaveBeenNthCalledWith(2, 'tail')
|
||||
})
|
||||
|
||||
it('stops a chunked write when the PTY incarnation changes between chunks', async () => {
|
||||
vi.useFakeTimers()
|
||||
const mockProc = createMockProc()
|
||||
spawnMock.mockReturnValue(mockProc.proc)
|
||||
registerPtyHandlers(mainWindow as never)
|
||||
const result = (await handlers.get('pty:spawn')!(null, {
|
||||
cols: 80,
|
||||
rows: 24
|
||||
})) as { id: string }
|
||||
restorePtyIncarnation(result.id, 'incarnation-old')
|
||||
const text = ['x'.repeat(TERMINAL_INPUT_CHUNK_MAX_BYTES), 'tail'].join('')
|
||||
|
||||
const writeResult = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, {
|
||||
id: result.id,
|
||||
data: text
|
||||
})
|
||||
expect(mockProc.proc.write).toHaveBeenCalledOnce()
|
||||
restorePtyIncarnation(result.id, 'incarnation-new')
|
||||
await vi.runAllTimersAsync()
|
||||
|
||||
await expect(writeResult).resolves.toBe(false)
|
||||
expect(mockProc.proc.write).toHaveBeenCalledOnce()
|
||||
clearProviderPtyState(result.id)
|
||||
})
|
||||
|
||||
it('yields while validating accepted large acknowledged pty writes before provider writes', async () => {
|
||||
const mockProc = createMockProc()
|
||||
spawnMock.mockReturnValue(mockProc.proc)
|
||||
@@ -18019,30 +17698,6 @@ describe('registerPtyHandlers', () => {
|
||||
expect(mockProc.proc.write.mock.calls.map(([chunk]) => chunk).join('')).toBe(text)
|
||||
})
|
||||
|
||||
it('drops a deferred write when the PTY incarnation changes during validation', async () => {
|
||||
vi.useFakeTimers()
|
||||
const mockProc = createMockProc()
|
||||
spawnMock.mockReturnValue(mockProc.proc)
|
||||
registerPtyHandlers(mainWindow as never)
|
||||
const result = (await handlers.get('pty:spawn')!(null, {
|
||||
cols: 80,
|
||||
rows: 24
|
||||
})) as { id: string }
|
||||
restorePtyIncarnation(result.id, 'incarnation-old')
|
||||
const text = 'é'.repeat(CLIPBOARD_TEXT_MEASURE_YIELD_CODE_UNITS + 1)
|
||||
|
||||
const writeResult = handlers.get('pty:writeAccepted')!(mainWindowIpcEvent, {
|
||||
id: result.id,
|
||||
data: text
|
||||
})
|
||||
restorePtyIncarnation(result.id, 'incarnation-new')
|
||||
await vi.runAllTimersAsync()
|
||||
|
||||
await expect(writeResult).resolves.toBe(false)
|
||||
expect(mockProc.proc.write).not.toHaveBeenCalled()
|
||||
clearProviderPtyState(result.id)
|
||||
})
|
||||
|
||||
it('rejects oversized acknowledged pty writes before provider writes', async () => {
|
||||
const mockProc = createMockProc()
|
||||
spawnMock.mockReturnValue(mockProc.proc)
|
||||
|
||||
+70
-234
@@ -15,8 +15,8 @@ export { getBashShellReadyRcfileContent } from '../providers/local-pty-shell-rea
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import type { PtyBindingSourceExpectation, Store } from '../persistence'
|
||||
import { retireTerminalSurfaceFromPersistence } from '../runtime/mobile-session-terminal-persistence-retirement'
|
||||
import { findTerminalTabIdForLeaf } from '../runtime/workspace-session-terminal-membership-authority'
|
||||
import type { GlobalSettings, TuiAgent } from '../../shared/types'
|
||||
import { toSshExecutionHostId } from '../../shared/execution-host'
|
||||
import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path'
|
||||
import { terminalOutputBacklogCapChars } from '../../shared/terminal-scrollback-policy'
|
||||
import type {
|
||||
@@ -85,7 +85,6 @@ import {
|
||||
import type { StartupCommandDelivery } from '../../shared/codex-startup-delivery'
|
||||
import {
|
||||
SSH_SESSION_EXPIRED_ERROR,
|
||||
isSshPtyExitedError,
|
||||
isSshPtyIdentityMismatchError,
|
||||
isSshPtyNotFoundError
|
||||
} from '../providers/ssh-pty-errors'
|
||||
@@ -237,7 +236,7 @@ import {
|
||||
} from '../project-groups/folder-workspace-path-status'
|
||||
import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch'
|
||||
import { resolveLocalProjectRuntimeForWorktreeId } from '../local-project-runtime-resolution'
|
||||
import { isPtyIncarnationId, isRelayAttestedPtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import { isPtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import type { PtyListedSession } from '../../shared/pty-listed-session'
|
||||
|
||||
// ─── Provider Registry ──────────────────────────────────────────────
|
||||
@@ -289,26 +288,6 @@ const ptyPaneKey = new Map<string, string>()
|
||||
// Why: reverse of ptyPaneKey — callers with a paneKey from outside the PTY lifecycle (e.g. agent-hook status routing) need the ptyId; kept in lock-step via the same sites.
|
||||
const paneKeyPtyId = new Map<string, string>()
|
||||
|
||||
/**
|
||||
* True only when this PTY's own pane has since bound a different PTY. The two
|
||||
* maps above are maintained in lock-step, so their disagreement is proof the id
|
||||
* was superseded — a renderer that queued work before a reattach cannot land it
|
||||
* on the successor.
|
||||
*
|
||||
* Deliberately false when no paneKey is recorded: an unowned or orphaned PTY is
|
||||
* unknown, not stale, and unknown never authorizes refusing an explicit
|
||||
* operation. That is also what keeps orphan cleanup working, since those ids
|
||||
* have no pane by construction.
|
||||
*/
|
||||
function isSupersededPtyId(ptyId: string): boolean {
|
||||
const paneKey = ptyPaneKey.get(ptyId)
|
||||
if (paneKey === undefined) {
|
||||
return false
|
||||
}
|
||||
const currentPtyId = paneKeyPtyId.get(paneKey)
|
||||
return currentPtyId !== undefined && currentPtyId !== ptyId
|
||||
}
|
||||
|
||||
const AGENT_HOOK_RUNTIME_ENV_KEYS = [
|
||||
'ORCA_AGENT_HOOK_PORT',
|
||||
'ORCA_AGENT_HOOK_TOKEN',
|
||||
@@ -538,68 +517,6 @@ function rememberPaneKeyForPty(ptyId: string, paneKey: unknown): string | null {
|
||||
return normalizedPaneKey
|
||||
}
|
||||
|
||||
/**
|
||||
* The one producer of a pane -> shell binding: durable record and fence maps together, keyed by the
|
||||
* tab holding the leaf *now* (a stored tabId names the tab a moved pane left). Splitting these let
|
||||
* the superseded-PTY fence sit inert on reattach, the path it was built for.
|
||||
*
|
||||
* `bound` is false when a durable pane refuses. A throw is unknown rather than a refusal, so it
|
||||
* propagates — only the caller that created the shell should clean it up.
|
||||
*
|
||||
* `tabId` is the resolved tab, returned so a caller that also registers the pane with the runtime
|
||||
* graph uses the SAME coordinate. Registering under the lease's frozen tabId while the record and
|
||||
* the fence use the live one splits the pane across two tabs, and the graph half then ensures a
|
||||
* mobile surface for the tab the pane left.
|
||||
*/
|
||||
/**
|
||||
* The tab a leaf lives in *now*. Callers whose own `tabId` is fresh (spawn) must not use this — the
|
||||
* persisted layout is the stale side inside the renderer's publish debounce. Callers holding a
|
||||
* tabId frozen in a durable lease (reattach) must, because the pane may have been moved since.
|
||||
*
|
||||
* Resolved separately from `bindPaneShell` so a thrown durable write cannot lose the answer and
|
||||
* leave the runtime graph registered under the tab the pane left.
|
||||
*/
|
||||
export function resolvePaneShellTabId(
|
||||
store: Pick<Store, 'getWorkspaceSession'> | undefined,
|
||||
leafId: string
|
||||
): string | undefined {
|
||||
if (typeof store?.getWorkspaceSession !== 'function') {
|
||||
return undefined
|
||||
}
|
||||
return findTerminalTabIdForLeaf(store.getWorkspaceSession(), leafId)
|
||||
}
|
||||
|
||||
export function bindPaneShell(args: {
|
||||
store: Pick<Store, 'persistPtyBinding' | 'getWorkspaceSession'> | undefined
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
incarnationId?: string
|
||||
startupCwd?: string
|
||||
mayCreate?: boolean
|
||||
expectedBinding?: { ptyId: string; incarnationId?: string }
|
||||
expectedSourceBinding?: PtyBindingSourceExpectation
|
||||
}): { bound: boolean; tabId: string } {
|
||||
const tabId = args.tabId
|
||||
const bound = args.store?.persistPtyBinding({
|
||||
worktreeId: args.worktreeId,
|
||||
tabId,
|
||||
leafId: args.leafId,
|
||||
ptyId: args.ptyId,
|
||||
...(args.incarnationId ? { incarnationId: args.incarnationId } : {}),
|
||||
...(args.startupCwd ? { startupCwd: args.startupCwd } : {}),
|
||||
...(args.mayCreate === false ? { mayCreate: false } : {}),
|
||||
...(args.expectedBinding ? { expectedBinding: args.expectedBinding } : {}),
|
||||
...(args.expectedSourceBinding ? { expectedSourceBinding: args.expectedSourceBinding } : {})
|
||||
})
|
||||
if (bound === false) {
|
||||
return { bound: false, tabId }
|
||||
}
|
||||
rememberPaneKeyForPty(args.ptyId, makePaneKey(tabId, args.leafId))
|
||||
return { bound: true, tabId }
|
||||
}
|
||||
|
||||
function cleanupPendingPaneSerializersForSender(ownerWebContentsId: number): void {
|
||||
pendingPaneSerializerCleanupRegistered.delete(ownerWebContentsId)
|
||||
for (const [paneKey, pending] of pendingByPaneKey) {
|
||||
@@ -736,13 +653,11 @@ type StablePaneAdoption = {
|
||||
} | null
|
||||
const stablePaneAdoptionsByOwnerKey = new Map<string, Promise<StablePaneAdoption>>()
|
||||
|
||||
// Pane bindings have one home: the local partition. An SSH pane is not partitioned out — the
|
||||
// renderer publishes its membership to `local` and `mayCreate: false` is evaluated there — so
|
||||
// selecting `ssh:<target>` here read a partition no live writer maintains (STA-3077 step P).
|
||||
function resolvePersistedStablePaneOwner(
|
||||
store: Store | undefined,
|
||||
paneKey: string,
|
||||
worktreeId: string
|
||||
worktreeId: string,
|
||||
connectionId: string | null | undefined
|
||||
): Pick<StablePaneOwner, 'tabId' | 'leafId' | 'ptyId' | 'incarnationId'> | null {
|
||||
if (!store || typeof store.getWorkspaceSession !== 'function') {
|
||||
return null
|
||||
@@ -751,7 +666,9 @@ function resolvePersistedStablePaneOwner(
|
||||
if (!parsed) {
|
||||
return null
|
||||
}
|
||||
const session = store.getWorkspaceSession()
|
||||
const session = store.getWorkspaceSession(
|
||||
connectionId ? toSshExecutionHostId(connectionId) : undefined
|
||||
)
|
||||
const tab = session.tabsByWorktree?.[worktreeId]?.find(
|
||||
(candidate) => candidate.id === parsed.tabId && candidate.worktreeId === worktreeId
|
||||
)
|
||||
@@ -791,7 +708,7 @@ function resolveStablePaneOwner(
|
||||
}
|
||||
}
|
||||
}
|
||||
const persisted = resolvePersistedStablePaneOwner(store, paneKey, worktreeId)
|
||||
const persisted = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId)
|
||||
if (resolved?.ptyId && persisted && resolved.ptyId !== persisted.ptyId) {
|
||||
throw new Error('terminal_pane_owner_conflict')
|
||||
}
|
||||
@@ -827,13 +744,15 @@ function resolveStablePaneOwner(
|
||||
function retirePersistedStablePaneOwner(
|
||||
store: Store | undefined,
|
||||
owner: StablePaneOwner,
|
||||
worktreeId: string
|
||||
worktreeId: string,
|
||||
connectionId: string | null | undefined
|
||||
): boolean {
|
||||
if (!store) {
|
||||
return false
|
||||
}
|
||||
const paneKey = makePaneKey(owner.tabId, owner.leafId)
|
||||
const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId)
|
||||
const hostId = connectionId ? toSshExecutionHostId(connectionId) : undefined
|
||||
const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId)
|
||||
if (!current) {
|
||||
// Why: persistence already dropped this pane binding (an earlier stop retired it while the
|
||||
// runtime kept history), so there is nothing left to clear — that is a completed retirement,
|
||||
@@ -843,7 +762,7 @@ function retirePersistedStablePaneOwner(
|
||||
if (current.ptyId !== owner.ptyId || current.incarnationId !== owner.persistedIncarnationId) {
|
||||
return false
|
||||
}
|
||||
const session = store.getWorkspaceSession()
|
||||
const session = store.getWorkspaceSession(hostId)
|
||||
const retired = retireTerminalSurfaceFromPersistence(session, {
|
||||
worktreeId,
|
||||
parentTabId: owner.tabId,
|
||||
@@ -854,7 +773,7 @@ function retirePersistedStablePaneOwner(
|
||||
if (retired === session) {
|
||||
return false
|
||||
}
|
||||
store.setWorkspaceSession(retired)
|
||||
store.setWorkspaceSession(retired, hostId)
|
||||
store.flushOrThrow()
|
||||
return true
|
||||
}
|
||||
@@ -869,8 +788,6 @@ type StablePaneSpawnContext = {
|
||||
connectionId?: string | null
|
||||
resolveOwner?: () => StablePaneOwner | null
|
||||
onFreshSpawn?: (result: PtySpawnResult) => void
|
||||
/** Create a shell for this pane instead of attaching the one it records; see the caller. */
|
||||
refuseAdoption?: boolean
|
||||
}
|
||||
|
||||
function stablePanePersistenceFence(
|
||||
@@ -890,22 +807,25 @@ function persistAdmittedStablePaneBinding(args: {
|
||||
result: PtySpawnResult
|
||||
worktreeId: string | undefined
|
||||
startupCwd: string | undefined
|
||||
connectionId: string | null | undefined
|
||||
}): boolean {
|
||||
const expectedBinding = stablePanePersistenceFence(args.owner)
|
||||
if (!args.store || !args.owner || !args.worktreeId || !expectedBinding) {
|
||||
return false
|
||||
}
|
||||
const persisted = bindPaneShell({
|
||||
store: args.store,
|
||||
worktreeId: args.worktreeId,
|
||||
tabId: args.owner.tabId,
|
||||
leafId: args.owner.leafId,
|
||||
ptyId: args.result.id,
|
||||
...(args.result.incarnationId ? { incarnationId: args.result.incarnationId } : {}),
|
||||
...(args.startupCwd ? { startupCwd: args.startupCwd } : {}),
|
||||
expectedBinding
|
||||
})
|
||||
if (persisted.bound === false) {
|
||||
const persisted = args.store.persistPtyBinding(
|
||||
{
|
||||
worktreeId: args.worktreeId,
|
||||
tabId: args.owner.tabId,
|
||||
leafId: args.owner.leafId,
|
||||
ptyId: args.result.id,
|
||||
...(args.result.incarnationId ? { incarnationId: args.result.incarnationId } : {}),
|
||||
...(args.startupCwd ? { startupCwd: args.startupCwd } : {}),
|
||||
expectedBinding
|
||||
},
|
||||
args.connectionId ? toSshExecutionHostId(args.connectionId) : undefined
|
||||
)
|
||||
if (persisted === false) {
|
||||
throw new Error('terminal_pane_owner_changed')
|
||||
}
|
||||
return true
|
||||
@@ -941,7 +861,7 @@ async function attachStablePaneOwner(
|
||||
if (isDaemonEndpointGoneError(error)) {
|
||||
throw new TerminalHostGoneError()
|
||||
}
|
||||
if (!isPtyProvenGoneForReplacement(error, owner.ptyId)) {
|
||||
if (!isPtyAlreadyGoneError(error)) {
|
||||
throw error
|
||||
}
|
||||
const ownerBeforeRetire = args.resolveOwner?.()
|
||||
@@ -957,7 +877,10 @@ async function attachStablePaneOwner(
|
||||
runtime?.onPtyExit(owner.ptyId, 0, owner.incarnationId)
|
||||
clearProviderPtyState(owner.ptyId)
|
||||
ptyOwnership.delete(owner.ptyId)
|
||||
if (args.worktreeId && !retirePersistedStablePaneOwner(args.store, owner, args.worktreeId)) {
|
||||
if (
|
||||
args.worktreeId &&
|
||||
!retirePersistedStablePaneOwner(args.store, owner, args.worktreeId, args.connectionId)
|
||||
) {
|
||||
throw new Error('terminal_pane_owner_changed')
|
||||
}
|
||||
if (args.resolveOwner?.()) {
|
||||
@@ -980,11 +903,7 @@ async function attachStablePaneOwner(
|
||||
async function spawnForStablePane(
|
||||
args: StablePaneSpawnContext
|
||||
): Promise<{ result: PtySpawnResult; owner: StablePaneOwner | null }> {
|
||||
// The one place an owner becomes `sessionId` for the provider, which is what makes an attach an
|
||||
// attach. Refusing adoption has to be honoured HERE: gating only where the owner is resolved
|
||||
// leaves every other resolution free to reach this line, which is exactly how the unreachable
|
||||
// pane's "start a new terminal" kept attaching the shell it could not reach.
|
||||
if (args.owner && !args.refuseAdoption) {
|
||||
if (args.owner) {
|
||||
const attached = await attachStablePaneOwner({ ...args, owner: args.owner })
|
||||
if (attached) {
|
||||
return attached
|
||||
@@ -1111,29 +1030,9 @@ function normalizeNodePtySpawnError(err: unknown): Error {
|
||||
return new Error(hintedMessage)
|
||||
}
|
||||
|
||||
/**
|
||||
* Retiring a pane's owner authorizes a replacement that carries its agent resume payload, so it
|
||||
* needs proof — unlike a shutdown, where "not found" is simply the outcome we asked for. A bare
|
||||
* not-found proves an exit only from a provider that owns its ptys; a replacement relay answers the
|
||||
* same for shells its predecessor is still running.
|
||||
*/
|
||||
function isPtyProvenGoneForReplacement(err: unknown, ptyId: string): boolean {
|
||||
if (!parseAppSshPtyId(ptyId)) {
|
||||
return isPtyAlreadyGoneError(err)
|
||||
}
|
||||
// For SSH, the two proving answers are the relay's own observed exit and the expiry the reattach
|
||||
// mints only after verifying that proof names this shell. A bare not-found is neither.
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
return isSshPtyExitedError(err) || message.includes(SSH_SESSION_EXPIRED_ERROR)
|
||||
}
|
||||
|
||||
function isPtyAlreadyGoneError(err: unknown): boolean {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
// A proven exit is the strongest form of "already gone"; it only reads differently because the
|
||||
// relay now says what it observed instead of reporting an unknown id.
|
||||
return (
|
||||
isSshPtyNotFoundError(err) || isSshPtyExitedError(err) || /Session not found/i.test(message)
|
||||
)
|
||||
return isSshPtyNotFoundError(err) || /Session not found/i.test(message)
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
@@ -5261,7 +5160,8 @@ export function registerPtyHandlers(
|
||||
owner: stablePaneOwner,
|
||||
result,
|
||||
worktreeId: hostSessionBinding?.worktreeId,
|
||||
startupCwd: cwd
|
||||
startupCwd: cwd,
|
||||
connectionId: args.connectionId
|
||||
})
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'terminal_pane_owner_changed') {
|
||||
@@ -5329,12 +5229,6 @@ export function registerPtyHandlers(
|
||||
...(typeof args.leafId === 'string' && isTerminalLeafId(args.leafId)
|
||||
? { leafId: args.leafId }
|
||||
: {}),
|
||||
// The shell this lease names, as the host just attested it. Without it the record says
|
||||
// only "pty-N", and a replaced relay reissues that from 1 — so reconnect would have
|
||||
// nothing to compare and would attach whatever now answers to the id.
|
||||
...(isRelayAttestedPtyIncarnationId(result.incarnationId)
|
||||
? { incarnationId: result.incarnationId }
|
||||
: {}),
|
||||
state: 'attached',
|
||||
lastAttachedAt: Date.now()
|
||||
})
|
||||
@@ -5358,8 +5252,7 @@ export function registerPtyHandlers(
|
||||
})
|
||||
if (hostSessionBinding && !stablePaneBindingPersisted) {
|
||||
try {
|
||||
const bound = bindPaneShell({
|
||||
store: hostSessionBinding.store,
|
||||
const binding = {
|
||||
worktreeId: hostSessionBinding.worktreeId,
|
||||
tabId: hostSessionBinding.tabId,
|
||||
leafId: hostSessionBinding.leafId,
|
||||
@@ -5369,8 +5262,14 @@ export function registerPtyHandlers(
|
||||
...(hostSessionBinding.expectedSourceBinding
|
||||
? { expectedSourceBinding: hostSessionBinding.expectedSourceBinding }
|
||||
: {})
|
||||
})
|
||||
if (bound.bound === false) {
|
||||
}
|
||||
const persisted = args.connectionId
|
||||
? hostSessionBinding.store.persistPtyBinding(
|
||||
binding,
|
||||
toSshExecutionHostId(args.connectionId)
|
||||
)
|
||||
: hostSessionBinding.store.persistPtyBinding(binding)
|
||||
if (persisted === false) {
|
||||
throw new Error('terminal_split_source_not_found')
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -5974,8 +5873,6 @@ export function registerPtyHandlers(
|
||||
cwd?: string
|
||||
// Why: fresh local spawns opt into recovering a saved cwd whose dir was deleted (#7239); reattach/remote need exact cwd, so the flag alone isn't sufficient.
|
||||
cwdFallback?: 'worktree'
|
||||
/** Create a shell for this pane rather than adopting the one it records. */
|
||||
createFreshShellForUnreachablePane?: boolean
|
||||
env?: Record<string, string>
|
||||
envToDelete?: string[]
|
||||
command?: string
|
||||
@@ -6075,14 +5972,8 @@ export function registerPtyHandlers(
|
||||
if (existingPaneSpawn) {
|
||||
return { ...(await existingPaneSpawn.promise), isReattach: true }
|
||||
}
|
||||
// Why the caller may refuse adoption: the unreachable-pane card offers "Start a new terminal"
|
||||
// for a pane whose recorded shell cannot be reached. Resolving an owner here makes the action
|
||||
// attach that shell first — and when it is unreachable the attach fails, the action creates
|
||||
// nothing, and the card comes back. The button is dead in the one state it is offered in.
|
||||
// Skipping the resolve is what makes it a creation: nothing is killed, and the old shell is
|
||||
// left alive and unbound for the cleanup surface.
|
||||
const earlyStablePaneOwner =
|
||||
earlyPaneKey && args.worktreeId && args.createFreshShellForUnreachablePane !== true
|
||||
earlyPaneKey && args.worktreeId
|
||||
? resolveStablePaneOwner(runtime, store, earlyPaneKey, args.worktreeId, args.connectionId)
|
||||
: null
|
||||
const earlyWorktreeId = args.worktreeId
|
||||
@@ -6694,7 +6585,6 @@ export function registerPtyHandlers(
|
||||
provider,
|
||||
spawnOptions,
|
||||
owner: stablePaneOwnerCandidate,
|
||||
refuseAdoption: args.createFreshShellForUnreachablePane === true,
|
||||
worktreeId: args.worktreeId,
|
||||
connectionId: args.connectionId,
|
||||
resolveOwner: () =>
|
||||
@@ -6834,7 +6724,8 @@ export function registerPtyHandlers(
|
||||
owner: stablePaneOwner,
|
||||
result,
|
||||
worktreeId: args.worktreeId,
|
||||
startupCwd: cwd
|
||||
startupCwd: cwd,
|
||||
connectionId: args.connectionId
|
||||
})
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'terminal_pane_owner_changed') {
|
||||
@@ -6887,11 +6778,6 @@ export function registerPtyHandlers(
|
||||
...(typeof args.worktreeId === 'string' ? { worktreeId: args.worktreeId } : {}),
|
||||
...(typeof args.tabId === 'string' ? { tabId: args.tabId } : {}),
|
||||
...(validatedLeafId ? { leafId: validatedLeafId } : {}),
|
||||
// See the sibling writer: a lease that names only "pty-N" cannot survive a relay that
|
||||
// reissues ids from 1.
|
||||
...(isRelayAttestedPtyIncarnationId(result.incarnationId)
|
||||
? { incarnationId: result.incarnationId }
|
||||
: {}),
|
||||
state: 'attached',
|
||||
lastAttachedAt: Date.now()
|
||||
})
|
||||
@@ -6912,15 +6798,19 @@ export function registerPtyHandlers(
|
||||
!stablePaneBindingPersisted
|
||||
) {
|
||||
try {
|
||||
bindPaneShell({
|
||||
store,
|
||||
const binding = {
|
||||
worktreeId: args.worktreeId,
|
||||
tabId: args.tabId,
|
||||
leafId: validatedLeafId,
|
||||
ptyId: result.id,
|
||||
...(result.incarnationId ? { incarnationId: result.incarnationId } : {}),
|
||||
...(cwd ? { startupCwd: cwd } : {})
|
||||
})
|
||||
}
|
||||
if (args.connectionId) {
|
||||
store.persistPtyBinding(binding, toSshExecutionHostId(args.connectionId))
|
||||
} else {
|
||||
store.persistPtyBinding(binding)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[pty] failed to persist PTY binding after spawn:', err)
|
||||
if (!result.isReattach) {
|
||||
@@ -7189,20 +7079,11 @@ export function registerPtyHandlers(
|
||||
mainWindow.webContents.send('pty:writeUnavailable', { id })
|
||||
}
|
||||
|
||||
type PtyWriteFence = { incarnationId: string | undefined }
|
||||
|
||||
const isCurrentPtyWrite = (id: string, fence: PtyWriteFence): boolean =>
|
||||
!isSupersededPtyId(id) && ptyIncarnationById.get(id) === fence.incarnationId
|
||||
|
||||
const writePtyProviderInputWithinLimit = (
|
||||
provider: IPtyProvider,
|
||||
id: string,
|
||||
data: string,
|
||||
fence: PtyWriteFence
|
||||
data: string
|
||||
): boolean | Promise<boolean> => {
|
||||
if (!isCurrentPtyWrite(id, fence)) {
|
||||
return false
|
||||
}
|
||||
const chunks = iterateTerminalInputChunks(data)
|
||||
const first = chunks.next()
|
||||
if (first.done) {
|
||||
@@ -7214,24 +7095,21 @@ export function registerPtyHandlers(
|
||||
provider.write(id, first.value)
|
||||
return true
|
||||
}
|
||||
return writePtyProviderInputChunks(provider, id, chunks, first.value, second.value, fence)
|
||||
return writePtyProviderInputChunks(provider, id, chunks, first.value, second.value)
|
||||
}
|
||||
|
||||
const writePtyProviderInput = (
|
||||
provider: IPtyProvider,
|
||||
id: string,
|
||||
data: string,
|
||||
fence: PtyWriteFence
|
||||
data: string
|
||||
): boolean | Promise<boolean> => {
|
||||
try {
|
||||
const tooLarge = isTerminalInputTooLargeWithDeferredMeasurement(data)
|
||||
if (typeof tooLarge === 'boolean') {
|
||||
return tooLarge ? false : writePtyProviderInputWithinLimit(provider, id, data, fence)
|
||||
return tooLarge ? false : writePtyProviderInputWithinLimit(provider, id, data)
|
||||
}
|
||||
return tooLarge
|
||||
.then((result) =>
|
||||
result ? false : writePtyProviderInputWithinLimit(provider, id, data, fence)
|
||||
)
|
||||
.then((result) => (result ? false : writePtyProviderInputWithinLimit(provider, id, data)))
|
||||
.catch((error) => {
|
||||
reportUnavailablePtyWrite(id, error)
|
||||
return false
|
||||
@@ -7247,16 +7125,12 @@ export function registerPtyHandlers(
|
||||
id: string,
|
||||
chunks: Iterator<string>,
|
||||
firstChunk: string,
|
||||
secondChunk: string,
|
||||
fence: PtyWriteFence
|
||||
secondChunk: string
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
let chunk: IteratorResult<string> = { done: false, value: firstChunk }
|
||||
let nextChunk: IteratorResult<string> = { done: false, value: secondChunk }
|
||||
while (!chunk.done) {
|
||||
if (!isCurrentPtyWrite(id, fence)) {
|
||||
return false
|
||||
}
|
||||
provider.write(id, chunk.value)
|
||||
if (!nextChunk.done) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
@@ -7301,17 +7175,11 @@ export function registerPtyHandlers(
|
||||
!mainWindow.isDestroyed() &&
|
||||
!(typeof mainWebContents.isDestroyed === 'function' && mainWebContents.isDestroyed())
|
||||
|
||||
const writePtyInput = (
|
||||
args: PtyWritePayload,
|
||||
fence: PtyWriteFence
|
||||
): boolean | Promise<boolean> => {
|
||||
const writePtyInput = (args: PtyWritePayload): boolean | Promise<boolean> => {
|
||||
// Why: mobile-presence-lock defense-in-depth — the renderer's onData guard can let one keystroke slip during the state-flip lag, so catch it server-side. See docs/mobile-presence-lock.md.
|
||||
if (runtime?.getDriver(args.id).kind === 'mobile') {
|
||||
return false
|
||||
}
|
||||
if (!isCurrentPtyWrite(args.id, fence)) {
|
||||
return false
|
||||
}
|
||||
const provider = ptyOwnership.has(args.id) ? tryGetProviderForPty(args.id) : undefined
|
||||
if (!provider) {
|
||||
return false
|
||||
@@ -7323,22 +7191,16 @@ export function registerPtyHandlers(
|
||||
if (visibleRendererPtys.has(args.id)) {
|
||||
clearHiddenRendererResizeOutput(args.id)
|
||||
}
|
||||
return writePtyProviderInput(provider, args.id, args.data, fence)
|
||||
return writePtyProviderInput(provider, args.id, args.data)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const writePtyInputAccepted = (
|
||||
args: PtyWritePayload,
|
||||
fence: PtyWriteFence
|
||||
): boolean | Promise<boolean> => {
|
||||
const writePtyInputAccepted = (args: PtyWritePayload): boolean | Promise<boolean> => {
|
||||
if (runtime?.getDriver(args.id).kind === 'mobile') {
|
||||
return false
|
||||
}
|
||||
if (!isCurrentPtyWrite(args.id, fence)) {
|
||||
return false
|
||||
}
|
||||
// Why: the ack infers Ctrl+C/Escape reached the local PTY; SSH providers are fire-and-forget relay notifications and can't truthfully acknowledge yet.
|
||||
if (ptyOwnership.get(args.id) !== null) {
|
||||
return false
|
||||
@@ -7354,7 +7216,7 @@ export function registerPtyHandlers(
|
||||
if (visibleRendererPtys.has(args.id)) {
|
||||
clearHiddenRendererResizeOutput(args.id)
|
||||
}
|
||||
return writePtyProviderInput(provider, args.id, args.data, fence)
|
||||
return writePtyProviderInput(provider, args.id, args.data)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
@@ -7366,31 +7228,21 @@ export function registerPtyHandlers(
|
||||
if (!isPtyWriteEventFromMainWindow(event, mainWindow.webContents) || !isPtyWritePayload(args)) {
|
||||
return
|
||||
}
|
||||
// Why here and not in the renderer: input queued before a reattach would
|
||||
// otherwise land on whatever PTY now holds the pane.
|
||||
const fence: PtyWriteFence = { incarnationId: ptyIncarnationById.get(args.id) }
|
||||
if (!isCurrentPtyWrite(args.id, fence)) {
|
||||
return
|
||||
}
|
||||
const claimTail = hostViewportClaimTails.get(args.id)
|
||||
if (claimTail) {
|
||||
void claimTail.then((claimed) => (claimed ? writePtyInput(args, fence) : false))
|
||||
void claimTail.then((claimed) => (claimed ? writePtyInput(args) : false))
|
||||
return
|
||||
}
|
||||
writePtyInput(args, fence)
|
||||
writePtyInput(args)
|
||||
})
|
||||
ipcMain.handle('pty:writeAccepted', (event, args: unknown): boolean | Promise<boolean> => {
|
||||
if (!isPtyWriteEventFromMainWindow(event, mainWindow.webContents) || !isPtyWritePayload(args)) {
|
||||
return false
|
||||
}
|
||||
const fence: PtyWriteFence = { incarnationId: ptyIncarnationById.get(args.id) }
|
||||
if (!isCurrentPtyWrite(args.id, fence)) {
|
||||
return false
|
||||
}
|
||||
const claimTail = hostViewportClaimTails.get(args.id)
|
||||
return claimTail
|
||||
? claimTail.then((claimed) => (claimed ? writePtyInputAccepted(args, fence) : false))
|
||||
: writePtyInputAccepted(args, fence)
|
||||
? claimTail.then((claimed) => (claimed ? writePtyInputAccepted(args) : false))
|
||||
: writePtyInputAccepted(args)
|
||||
})
|
||||
|
||||
ipcMain.removeAllListeners('pty:claimViewport')
|
||||
@@ -7423,10 +7275,6 @@ export function registerPtyHandlers(
|
||||
// Why: resize is fire-and-forget — ipcMain.on (not .handle) halves IPC traffic by skipping the empty acknowledgement reply.
|
||||
ipcMain.removeAllListeners('pty:resize')
|
||||
ipcMain.on('pty:resize', (_event, args: { id: string; cols: number; rows: number }) => {
|
||||
// Why: a resize for a pane that has rebound would reshape the successor's shell.
|
||||
if (isSupersededPtyId(args.id)) {
|
||||
return
|
||||
}
|
||||
// Why: after a desktop-fit override change the renderer's safeFit cascade re-measures ALL panes (background ones at full width), so suppress every pty:resize in this window to avoid corrupting PTY dimensions.
|
||||
if (runtime?.isResizeSuppressed()) {
|
||||
return
|
||||
@@ -7691,15 +7539,6 @@ export function registerPtyHandlers(
|
||||
|
||||
ipcMain.removeAllListeners('pty:signal')
|
||||
ipcMain.on('pty:signal', (_event, args: { id: string; signal: string }) => {
|
||||
// Why fenced but pty:kill is not: a signal means "interrupt MY pane", so a
|
||||
// superseded id is a misdirected interrupt. A kill on a superseded id is the
|
||||
// opposite — that PTY is now orphaned and reclaiming it is the point.
|
||||
if (isSupersededPtyId(args.id)) {
|
||||
return
|
||||
}
|
||||
// Routing refuses a session whose host is unreachable, but sendSignal is async everywhere,
|
||||
// so that refusal arrives as a rejection rather than a throw — and optional chaining
|
||||
// short-circuits the whole chain when there is no provider at all.
|
||||
tryGetProviderForPty(args.id)
|
||||
?.sendSignal(args.id, args.signal)
|
||||
.catch(() => {})
|
||||
@@ -7852,9 +7691,6 @@ export function registerPtyHandlers(
|
||||
}
|
||||
const ownedConnectionId = ptyOwnership.get(args.id)
|
||||
const parsedSshId = ownedConnectionId === undefined ? parseAppSshPtyId(args.id) : null
|
||||
// Why: before the cold-start daemon swap lands, the pre-swap local provider owns no
|
||||
// daemon id and would answer an authoritative false for every restored session.
|
||||
await getLocalPtyProviderStartupPromise(ownedConnectionId ?? parsedSshId?.connectionId)
|
||||
const provider = parsedSshId
|
||||
? sshProviders.get(parsedSshId.connectionId)
|
||||
: tryGetProviderForPty(args.id)
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ import {
|
||||
SshPtyOutputIntake,
|
||||
type SshPtyOutputDataEvent,
|
||||
type SshPtyOutputIntakeDependencies
|
||||
} from '../ssh-pty-output-intake'
|
||||
} from './ssh-pty-output-intake'
|
||||
|
||||
export function sshPtyOutputEvent(
|
||||
overrides: Partial<SshPtyOutputDataEvent> = {}
|
||||
@@ -3,7 +3,7 @@ import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import {
|
||||
createSshPtyOutputIntakeHarness as createHarness,
|
||||
sshPtyOutputEvent as event
|
||||
} from './__tests__/ssh-pty-output-intake-test-harness'
|
||||
} from './ssh-pty-output-intake-test-harness'
|
||||
|
||||
describe('SshPtyOutputIntake', () => {
|
||||
it('plateaus at the model and pressure budgets, then resumes below low water', async () => {
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { SshPtyOutputDataEvent } from './ssh-pty-output-intake'
|
||||
import {
|
||||
createSshPtyOutputIntakeHarness as createHarness,
|
||||
sshPtyOutputEvent as event
|
||||
} from './__tests__/ssh-pty-output-intake-test-harness'
|
||||
} from './ssh-pty-output-intake-test-harness'
|
||||
|
||||
// Production-shaped ids: bare relay id on the wire side, prefixed app id in intake.
|
||||
const APP_ID = 'ssh:conn@@pty-1'
|
||||
|
||||
@@ -305,7 +305,6 @@ describe('SSH IPC handlers', () => {
|
||||
upsertSshPtyConsumerRecovery: vi.fn(),
|
||||
removeSshPtyConsumerRecovery: vi.fn(),
|
||||
getSshRemotePtyLeases: vi.fn().mockReturnValue([]),
|
||||
supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0),
|
||||
markSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLeases: vi.fn(),
|
||||
markSshRemotePtyLeasesAsync: vi.fn(),
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { getDefaultPersistedState } from '../shared/constants'
|
||||
|
||||
const testState = { dir: '' }
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: { getPath: () => testState.dir },
|
||||
safeStorage: { isEncryptionAvailable: () => false }
|
||||
}))
|
||||
vi.mock('./telemetry/client', () => ({ track: vi.fn() }))
|
||||
vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() }))
|
||||
|
||||
const TARGET = 'ssh-target-1'
|
||||
|
||||
type DurableFlushStore = {
|
||||
flushDurableStateOrThrowAsync: (drainToStableGeneration?: boolean) => Promise<void>
|
||||
}
|
||||
|
||||
function lease(ptyId: string, state: 'attached' | 'detached') {
|
||||
return {
|
||||
targetId: TARGET,
|
||||
ptyId,
|
||||
state,
|
||||
createdAt: 1,
|
||||
updatedAt: 1
|
||||
}
|
||||
}
|
||||
|
||||
async function createStore() {
|
||||
mkdirSync(testState.dir, { recursive: true })
|
||||
writeFileSync(
|
||||
join(testState.dir, 'orca-data.json'),
|
||||
JSON.stringify({
|
||||
...getDefaultPersistedState(testState.dir),
|
||||
sshRemotePtyLeases: [
|
||||
lease('pty-1', 'detached'),
|
||||
lease('pty-2', 'attached'),
|
||||
lease('pty-3', 'detached')
|
||||
]
|
||||
}),
|
||||
'utf-8'
|
||||
)
|
||||
vi.resetModules()
|
||||
const { Store, initDataPath } = await import('./persistence')
|
||||
initDataPath()
|
||||
return new Store()
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
testState.dir = mkdtempSync(join(tmpdir(), 'orca-ssh-lease-termination-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(testState.dir, { force: true, recursive: true })
|
||||
})
|
||||
|
||||
describe('SSH lease termination persistence', () => {
|
||||
it('updates only selected leases and awaits one asynchronous durability barrier', async () => {
|
||||
const store = await createStore()
|
||||
const asyncFlush = vi
|
||||
.spyOn(store as unknown as DurableFlushStore, 'flushDurableStateOrThrowAsync')
|
||||
.mockResolvedValue()
|
||||
const syncFlush = vi.spyOn(store, 'flush')
|
||||
const syncFlushOrThrow = vi.spyOn(store, 'flushOrThrow')
|
||||
|
||||
await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1', 'pty-3'])
|
||||
|
||||
expect(asyncFlush).toHaveBeenCalledOnce()
|
||||
expect(asyncFlush).toHaveBeenCalledWith(false)
|
||||
expect(syncFlush).not.toHaveBeenCalled()
|
||||
expect(syncFlushOrThrow).not.toHaveBeenCalled()
|
||||
expect(store.getSshRemotePtyLeases(TARGET)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }),
|
||||
expect.objectContaining({ ptyId: 'pty-2', state: 'attached' }),
|
||||
expect.objectContaining({ ptyId: 'pty-3', state: 'terminated' })
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('does not overwrite concurrent lease state when durability fails', async () => {
|
||||
const store = await createStore()
|
||||
const writeError = new Error('disk full')
|
||||
const asyncFlush = vi
|
||||
.spyOn(store as unknown as DurableFlushStore, 'flushDurableStateOrThrowAsync')
|
||||
.mockRejectedValue(writeError)
|
||||
const scheduleSave = vi.spyOn(store as unknown as { scheduleSave: () => void }, 'scheduleSave')
|
||||
scheduleSave.mockClear()
|
||||
|
||||
const retirement = store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1'])
|
||||
store.markSshRemotePtyLeasesForShutdown(TARGET, 'detached')
|
||||
|
||||
await expect(retirement).rejects.toBe(writeError)
|
||||
expect(scheduleSave).toHaveBeenCalledTimes(2)
|
||||
expect(store.getSshRemotePtyLeases(TARGET)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' }),
|
||||
expect.objectContaining({ ptyId: 'pty-2', state: 'detached' })
|
||||
])
|
||||
)
|
||||
|
||||
asyncFlush.mockRestore()
|
||||
await store.flushPendingOrThrowAsync({ drainToStableGeneration: false })
|
||||
const persisted = JSON.parse(readFileSync(join(testState.dir, 'orca-data.json'), 'utf-8')) as {
|
||||
sshRemotePtyLeases: { ptyId: string; state: string }[]
|
||||
}
|
||||
expect(persisted.sshRemotePtyLeases).toContainEqual(
|
||||
expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' })
|
||||
)
|
||||
})
|
||||
|
||||
it('does not write when no requested lease exists', async () => {
|
||||
const store = await createStore()
|
||||
const asyncFlush = vi.spyOn(
|
||||
store as unknown as DurableFlushStore,
|
||||
'flushDurableStateOrThrowAsync'
|
||||
)
|
||||
|
||||
await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['missing'])
|
||||
|
||||
expect(asyncFlush).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not chase mutations made after the retirement generation is durable', async () => {
|
||||
const store = await createStore()
|
||||
const internal = store as unknown as { enqueueWrite: () => Promise<void> }
|
||||
const enqueueWrite = internal.enqueueWrite.bind(store)
|
||||
let injectedMutation = false
|
||||
const enqueueSpy = vi.spyOn(internal, 'enqueueWrite').mockImplementation(async () => {
|
||||
await enqueueWrite()
|
||||
if (!injectedMutation) {
|
||||
injectedMutation = true
|
||||
store.updateUI({ sidebarWidth: 777 })
|
||||
}
|
||||
})
|
||||
|
||||
await store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['pty-1'])
|
||||
|
||||
expect(enqueueSpy).toHaveBeenCalledOnce()
|
||||
const persisted = JSON.parse(readFileSync(join(testState.dir, 'orca-data.json'), 'utf-8')) as {
|
||||
sshRemotePtyLeases: { ptyId: string; state: string }[]
|
||||
}
|
||||
expect(persisted.sshRemotePtyLeases).toContainEqual(
|
||||
expect.objectContaining({ ptyId: 'pty-1', state: 'terminated' })
|
||||
)
|
||||
enqueueSpy.mockRestore()
|
||||
await store.flushPendingOrThrowAsync({ drainToStableGeneration: false })
|
||||
})
|
||||
})
|
||||
@@ -10031,9 +10031,6 @@ describe('Store', () => {
|
||||
expect(store.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey?.[paneKey]).toBe(
|
||||
'inc-live'
|
||||
)
|
||||
// Unchanged by STA-3077 step P. The fold moves an SSH pane's incarnation only when it moves
|
||||
// the binding it fences, and this fixture's tab exists solely in the SSH partition — so
|
||||
// there is no second home to disagree with, nothing folds, and both halves stay together.
|
||||
const reloaded = await createStore()
|
||||
expect(reloaded.getWorkspaceSession(hostId).terminalPtyIncarnationsByPaneKey?.[paneKey]).toBe(
|
||||
'inc-live'
|
||||
|
||||
+7
-382
@@ -37,7 +37,6 @@ import {
|
||||
nextAutomationOccurrenceAfter
|
||||
} from '../shared/automation-schedules'
|
||||
import { getAutomationLegacyRepoId } from '../shared/automation-run-identity'
|
||||
import { isRelayAttestedPtyIncarnationId } from '../shared/pty-incarnation'
|
||||
import { normalizeAutomationPrecheck } from '../shared/automation-precheck'
|
||||
import { normalizeProxyUrl } from '../shared/network-proxy'
|
||||
import { normalizeKagiSessionLink } from '../shared/browser-url'
|
||||
@@ -97,7 +96,6 @@ import type { MigrationUnsupportedPtyEntry } from '../shared/agent-status-types'
|
||||
import { MOBILE_PAIRING_USERDATA_FILES } from './runtime/mobile-pairing-files'
|
||||
import { normalizePersistedMobileClientTabSelections } from './runtime/client-session-tab-selection-persistence'
|
||||
import { sanitizeWorkspaceSessionTerminalRetirements } from './runtime/mobile-session-terminal-persistence-retirement'
|
||||
import { findTerminalTabIdForLeaf } from './runtime/workspace-session-terminal-membership-authority'
|
||||
import {
|
||||
removeRepoFromHostWorkspaceSessions,
|
||||
removeRepoFromWorkspaceSession
|
||||
@@ -1660,13 +1658,7 @@ function normalizeSshRemotePtyLease(value: unknown): SshRemotePtyLease | null {
|
||||
createdAt: typeof raw.createdAt === 'number' ? raw.createdAt : now,
|
||||
updatedAt: typeof raw.updatedAt === 'number' ? raw.updatedAt : now,
|
||||
...(typeof raw.lastAttachedAt === 'number' ? { lastAttachedAt: raw.lastAttachedAt } : {}),
|
||||
...(typeof raw.lastDetachedAt === 'number' ? { lastDetachedAt: raw.lastDetachedAt } : {}),
|
||||
// Rebuilt field by field, so a lease is only as durable as this list — adding the field to the
|
||||
// type without adding it here drops it on every load. Synthesized values are refused rather
|
||||
// than carried: one written by an older build would later read as a different shell.
|
||||
...(isRelayAttestedPtyIncarnationId(raw.incarnationId)
|
||||
? { incarnationId: raw.incarnationId }
|
||||
: {})
|
||||
...(typeof raw.lastDetachedAt === 'number' ? { lastDetachedAt: raw.lastDetachedAt } : {})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1767,20 +1759,6 @@ function layoutContainsLeafId(node: TerminalPaneLayoutNode | null, leafId: strin
|
||||
return layoutContainsLeafId(node.first, leafId) || layoutContainsLeafId(node.second, leafId)
|
||||
}
|
||||
|
||||
/** Total order so two leases for one pane resolve the same way on every host. */
|
||||
function isNewerSshRemotePtyLease(
|
||||
candidate: SshRemotePtyLease,
|
||||
incumbent: SshRemotePtyLease
|
||||
): boolean {
|
||||
if (candidate.updatedAt !== incumbent.updatedAt) {
|
||||
return candidate.updatedAt > incumbent.updatedAt
|
||||
}
|
||||
if (candidate.createdAt !== incumbent.createdAt) {
|
||||
return candidate.createdAt > incumbent.createdAt
|
||||
}
|
||||
return candidate.ptyId > incumbent.ptyId
|
||||
}
|
||||
|
||||
function cloneLayoutNode(node: TerminalPaneLayoutNode): TerminalPaneLayoutNode {
|
||||
if (node.type === 'leaf') {
|
||||
return { type: 'leaf', leafId: node.leafId }
|
||||
@@ -2876,7 +2854,6 @@ export class Store {
|
||||
private pendingGithubCacheWrite: Promise<void> | null = null
|
||||
private readonly staleGithubCacheTempCleanup: Promise<void>
|
||||
private gitUsernameCache = new Map<string, string>()
|
||||
private readonly sshRemotePtyLeaseMutationVersions = new WeakMap<SshRemotePtyLease, number>()
|
||||
private readonly protectedSecrets = new ProtectedSecretPersistence()
|
||||
private loadNeedsSave = false
|
||||
private settingsChangeListeners = new Set<
|
||||
@@ -6918,14 +6895,10 @@ export class Store {
|
||||
incarnationId?: string
|
||||
startupCwd?: string
|
||||
expectedBinding?: { ptyId: string; incarnationId?: string }
|
||||
/** Reattach passes false: an absent durable pane never authorizes creating UI.
|
||||
* Defaults true so the spawn path keeps its force-quit-race branches. */
|
||||
mayCreate?: boolean
|
||||
expectedSourceBinding?: PtyBindingSourceExpectation
|
||||
},
|
||||
hostId?: string | null
|
||||
): boolean {
|
||||
const mayCreate = args.mayCreate ?? true
|
||||
const resolvedHostId = this.resolveHostId(hostId)
|
||||
const session = this.getWorkspaceSession(resolvedHostId)
|
||||
const paneKey = `${args.tabId}:${args.leafId}`
|
||||
@@ -7038,10 +7011,6 @@ export class Store {
|
||||
[bindingWorktreeId]: session.activeTabIdByWorktree?.[bindingWorktreeId] ?? args.tabId
|
||||
}
|
||||
}
|
||||
if (!mayCreate && terminalMembershipChanged) {
|
||||
restoreSession()
|
||||
return false
|
||||
}
|
||||
if (!isTerminalLeafId(args.leafId)) {
|
||||
// Why: keep legacy renderer-local pane ids out of durable leaf-keyed layout state after the UUID migration.
|
||||
advanceTopologyFence()
|
||||
@@ -7092,10 +7061,6 @@ export class Store {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!mayCreate && terminalMembershipChanged) {
|
||||
restoreSession()
|
||||
return false
|
||||
}
|
||||
advanceTopologyFence()
|
||||
try {
|
||||
this.flushOrThrow()
|
||||
@@ -7391,12 +7356,6 @@ export class Store {
|
||||
return leases.filter((lease) => targetId === undefined || lease.targetId === targetId)
|
||||
}
|
||||
|
||||
private advanceSshRemotePtyLeaseMutationVersion(lease: SshRemotePtyLease): number {
|
||||
const version = (this.sshRemotePtyLeaseMutationVersions.get(lease) ?? 0) + 1
|
||||
this.sshRemotePtyLeaseMutationVersions.set(lease, version)
|
||||
return version
|
||||
}
|
||||
|
||||
upsertSshRemotePtyLease(
|
||||
lease: Omit<SshRemotePtyLease, 'createdAt' | 'updatedAt'> &
|
||||
Partial<Pick<SshRemotePtyLease, 'createdAt' | 'updatedAt'>>
|
||||
@@ -7428,301 +7387,9 @@ export class Store {
|
||||
} else {
|
||||
this.state.sshRemotePtyLeases.push(next)
|
||||
}
|
||||
this.supersedeSiblingLeasesForPane(next, now)
|
||||
this.flush()
|
||||
}
|
||||
|
||||
/**
|
||||
* One pane owns at most one live remote PTY. Without this, lease identity is
|
||||
* `(targetId, ptyId)` alone, so a pane that re-leases under a new relay id
|
||||
* leaves its predecessor live forever — reattach then fans out over both and
|
||||
* grafts a pane the user never opened (STA-3077: 2 -> 19 -> 20 across three
|
||||
* reconnects).
|
||||
*
|
||||
* Superseded leases are marked `expired`, not terminated: the remote shell is
|
||||
* deliberately left running, because losing a lease is not proof the shell died.
|
||||
*/
|
||||
private supersedeSiblingLeasesForPane(winner: SshRemotePtyLease, now: number): void {
|
||||
if (!winner.worktreeId || !winner.tabId || !winner.leafId) {
|
||||
return
|
||||
}
|
||||
if (winner.state === 'terminated' || winner.state === 'expired') {
|
||||
return
|
||||
}
|
||||
// Why consult the binding here: at upsert time the caller's lease may not be
|
||||
// the one the pane is bound to yet. Expiring the bound predecessor would
|
||||
// detach a live pane, so leave both live and let reattach arbitrate with the
|
||||
// binding in hand.
|
||||
const boundPtyId = this.durablyBoundPtyIdForPane(winner.targetId, winner.tabId, winner.leafId)
|
||||
if (boundPtyId && boundPtyId !== winner.ptyId) {
|
||||
return
|
||||
}
|
||||
const superseded: SshRemotePtyLease[] = []
|
||||
for (const lease of this.state.sshRemotePtyLeases ?? []) {
|
||||
if (
|
||||
lease.ptyId !== winner.ptyId &&
|
||||
lease.targetId === winner.targetId &&
|
||||
lease.worktreeId === winner.worktreeId &&
|
||||
// Leaf only: a lease freezes its tabId, and a pane broken out to a new tab would otherwise
|
||||
// never compete with its own predecessor — which is the reported cardinality growth.
|
||||
lease.leafId === winner.leafId
|
||||
) {
|
||||
if (lease.state === 'expired') {
|
||||
// A concurrent same-pane owner confirms this lease must outlive rollback as retired.
|
||||
this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
continue
|
||||
}
|
||||
if (lease.state === 'terminated') {
|
||||
continue
|
||||
}
|
||||
lease.state = 'expired'
|
||||
lease.updatedAt = now
|
||||
this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
superseded.push(lease)
|
||||
}
|
||||
}
|
||||
// Why: matching on lease ptyId first means this scrubs only the predecessor's
|
||||
// stale binding — the winner's own binding cannot match and is left intact.
|
||||
this.clearSshRemotePtyBindingsForLeases(winner.targetId, superseded, 'local')
|
||||
}
|
||||
|
||||
/** The PTY a pane is durably bound to. The desktop plane's home is `local` — the renderer is its
|
||||
* only publisher of pane membership. Hedging into `ssh:<target>` let the headless plane's copy
|
||||
* outvote the live binding and silently no-op supersession (STA-3077).
|
||||
*
|
||||
* Keyed on the leaf, falling back across tabs: a lease freezes its tabId at write time, and
|
||||
* `detachTerminalPaneToTab` moves a live pane, so the named tab can be the one the pane left.
|
||||
* Missing the binding there makes it lose to recency and retires the pane's own shell. */
|
||||
private durablyBoundPtyIdForPane(
|
||||
targetId: string,
|
||||
tabId: string,
|
||||
leafId: string
|
||||
): string | undefined {
|
||||
// Deliberately does NOT require the tab to still exist, unlike findTerminalTabIdForLeaf, which
|
||||
// must — binding a live shell to a deleted tab registers a ghost pane. Here the consequence
|
||||
// runs the other way: a binding under a departed tab only ever keeps a lease alive, and not
|
||||
// retiring is the safe direction when nothing has proved the shell dead.
|
||||
const findLeafBinding = (session: WorkspaceSessionState | undefined): string | undefined => {
|
||||
const layouts = session?.terminalLayoutsByTabId
|
||||
return (
|
||||
layouts?.[tabId]?.ptyIdsByLeafId?.[leafId] ??
|
||||
Object.values(layouts ?? {}).find((layout) => layout?.ptyIdsByLeafId?.[leafId])
|
||||
?.ptyIdsByLeafId?.[leafId]
|
||||
)
|
||||
}
|
||||
// Local FIRST, and only then the headless plane. Local-first is what STA-3077 needed: the old
|
||||
// code preferred `ssh:<target>`, so a copy no live writer maintained outvoted the real binding
|
||||
// and supersession no-opped. The fallback is not that hedge — it only speaks for a pane local
|
||||
// says nothing about, so a headless-owned pane still gets a vote before its lease is retired.
|
||||
const boundPtyId =
|
||||
findLeafBinding(this.state.workspaceSession) ??
|
||||
findLeafBinding(this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)])
|
||||
return boundPtyId ? this.getRelayPtyIdForSshLeaseComparison(targetId, boundPtyId) : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* The durable pane binding outranks recency. Picking the newest lease alone
|
||||
* would retire the one the pane is actually bound to whenever a newer unbound
|
||||
* lease exists, which detaches a live pane instead of healing it.
|
||||
*/
|
||||
private outranksForPane(
|
||||
candidate: SshRemotePtyLease,
|
||||
incumbent: SshRemotePtyLease,
|
||||
targetId: string
|
||||
): boolean {
|
||||
const boundPtyId = this.durablyBoundPtyIdForPane(
|
||||
targetId,
|
||||
candidate.tabId ?? '',
|
||||
candidate.leafId ?? ''
|
||||
)
|
||||
if (boundPtyId) {
|
||||
if (incumbent.ptyId === boundPtyId) {
|
||||
return false
|
||||
}
|
||||
if (candidate.ptyId === boundPtyId) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return isNewerSshRemotePtyLease(candidate, incumbent)
|
||||
}
|
||||
|
||||
/**
|
||||
* Heals lease state that predates pane-keyed supersession. Existing installs
|
||||
* carry the duplicates STA-3077 accumulated — one report reached 20 live
|
||||
* leases for a handful of panes — and supersession alone only prevents new
|
||||
* ones. Reattach calls this first so a stale predecessor cannot be revived.
|
||||
*
|
||||
* Returns the number of leases retired, for logging.
|
||||
*/
|
||||
async supersedeDuplicatePaneLeases(targetId: string): Promise<number> {
|
||||
const live = (this.state.sshRemotePtyLeases ?? []).filter(
|
||||
(lease) =>
|
||||
lease.targetId === targetId && lease.state !== 'terminated' && lease.state !== 'expired'
|
||||
)
|
||||
const winnerByPane = new Map<string, SshRemotePtyLease>()
|
||||
for (const lease of live) {
|
||||
if (!lease.worktreeId || !lease.tabId || !lease.leafId) {
|
||||
continue
|
||||
}
|
||||
// Keyed on the leaf, not the frozen tabId: the same pane moved between tabs must land in one
|
||||
// bucket, or its duplicates never arbitrate against each other.
|
||||
const paneKey = [lease.worktreeId, lease.leafId].join('\0')
|
||||
const incumbent = winnerByPane.get(paneKey)
|
||||
if (!incumbent || this.outranksForPane(lease, incumbent, targetId)) {
|
||||
winnerByPane.set(paneKey, lease)
|
||||
}
|
||||
}
|
||||
if (winnerByPane.size === 0) {
|
||||
return 0
|
||||
}
|
||||
const winners = new Set(winnerByPane.values())
|
||||
const now = Date.now()
|
||||
const superseded: SshRemotePtyLease[] = []
|
||||
const restore: (() => void)[] = []
|
||||
for (const lease of live) {
|
||||
if (!lease.worktreeId || !lease.tabId || !lease.leafId || winners.has(lease)) {
|
||||
continue
|
||||
}
|
||||
const { state, updatedAt } = lease
|
||||
lease.state = 'expired'
|
||||
lease.updatedAt = now
|
||||
const rollbackVersion = this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
restore.push(() => {
|
||||
if (
|
||||
this.state.sshRemotePtyLeases?.includes(lease) &&
|
||||
this.sshRemotePtyLeaseMutationVersions.get(lease) === rollbackVersion &&
|
||||
lease.state === 'expired' &&
|
||||
lease.updatedAt === now
|
||||
) {
|
||||
lease.state = state
|
||||
lease.updatedAt = updatedAt
|
||||
this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
}
|
||||
})
|
||||
superseded.push(lease)
|
||||
}
|
||||
if (superseded.length === 0) {
|
||||
return 0
|
||||
}
|
||||
const bindingsBefore = this.snapshotSshLeaseBindings(targetId, superseded)
|
||||
this.clearSshRemotePtyBindingsForLeases(targetId, superseded, 'local')
|
||||
this.scheduleSave()
|
||||
try {
|
||||
// Why the ASYNC twin: the sync flush fsyncs a multi-MB file from the Electron main thread,
|
||||
// and this runs on every reconnect. On a stalled network profile mount that syscall is
|
||||
// uninterruptible and nothing can bound it — see flushAsync. Why "OrThrow" and not plain
|
||||
// flush(): flush() swallows write errors, which would leave these leases retired in memory
|
||||
// but attached on disk for the rest of the session.
|
||||
await this.flushDurableStateOrThrowAsync(false)
|
||||
} catch (err) {
|
||||
for (const undo of restore) {
|
||||
undo()
|
||||
}
|
||||
this.restoreSshLeaseBindings(targetId, superseded, bindingsBefore)
|
||||
this.scheduleSave()
|
||||
console.error('[persistence] Failed to retire duplicate pane leases:', err)
|
||||
return 0
|
||||
}
|
||||
return superseded.length
|
||||
}
|
||||
|
||||
private snapshotSshLeaseBindings(
|
||||
targetId: string,
|
||||
leases: SshRemotePtyLease[]
|
||||
): {
|
||||
tabs: { leafId: string; ptyId: string }[]
|
||||
leaves: { leafId: string; ptyId: string }[]
|
||||
} {
|
||||
const tabSnapshots: { leafId: string; ptyId: string }[] = []
|
||||
const leaves: { leafId: string; ptyId: string }[] = []
|
||||
const session = this.state.workspaceSession
|
||||
for (const [worktreeId, tabs] of Object.entries(session?.tabsByWorktree ?? {})) {
|
||||
for (const tab of tabs) {
|
||||
if (tab.ptyId) {
|
||||
const lease = leases.find((candidate) =>
|
||||
this.sshRemotePtyLeaseMayReferenceBinding(candidate, {
|
||||
ptyId: tab.ptyId!,
|
||||
worktreeId,
|
||||
targetId,
|
||||
tabId: tab.id
|
||||
})
|
||||
)
|
||||
if (lease?.leafId) {
|
||||
tabSnapshots.push({ leafId: lease.leafId, ptyId: tab.ptyId })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [tabId, layout] of Object.entries(session?.terminalLayoutsByTabId ?? {})) {
|
||||
const worktreeId = Object.entries(session?.tabsByWorktree ?? {}).find(([, tabs]) =>
|
||||
tabs.some((tab) => tab.id === tabId)
|
||||
)?.[0]
|
||||
for (const [leafId, ptyId] of Object.entries(layout.ptyIdsByLeafId ?? {})) {
|
||||
if (
|
||||
leases.some(
|
||||
(lease) =>
|
||||
lease.leafId === leafId &&
|
||||
this.sshRemotePtyLeaseMayReferenceBinding(lease, {
|
||||
ptyId,
|
||||
targetId,
|
||||
...(worktreeId ? { worktreeId } : {}),
|
||||
tabId,
|
||||
leafId
|
||||
})
|
||||
)
|
||||
) {
|
||||
leaves.push({ leafId, ptyId })
|
||||
}
|
||||
}
|
||||
}
|
||||
return { tabs: tabSnapshots, leaves }
|
||||
}
|
||||
|
||||
private restoreSshLeaseBindings(
|
||||
targetId: string,
|
||||
leases: SshRemotePtyLease[],
|
||||
snapshots: {
|
||||
tabs: { leafId: string; ptyId: string }[]
|
||||
leaves: { leafId: string; ptyId: string }[]
|
||||
}
|
||||
): void {
|
||||
const currentLeases = new Set(this.state.sshRemotePtyLeases ?? [])
|
||||
const session = this.state.workspaceSession
|
||||
const hasLiveLease = (snapshot: { leafId: string; ptyId: string }): boolean =>
|
||||
leases.some(
|
||||
(lease) =>
|
||||
currentLeases.has(lease) &&
|
||||
lease.state !== 'terminated' &&
|
||||
lease.state !== 'expired' &&
|
||||
lease.targetId === targetId &&
|
||||
lease.leafId === snapshot.leafId &&
|
||||
lease.ptyId === this.getRelayPtyIdForSshLeaseComparison(targetId, snapshot.ptyId)
|
||||
)
|
||||
for (const snapshot of snapshots.tabs) {
|
||||
const tabId = findTerminalTabIdForLeaf(session, snapshot.leafId)
|
||||
const tab = Object.values(session?.tabsByWorktree ?? {})
|
||||
.flat()
|
||||
.find((candidate) => candidate.id === tabId)
|
||||
if (hasLiveLease(snapshot) && tab?.ptyId === null) {
|
||||
tab.ptyId = snapshot.ptyId
|
||||
}
|
||||
}
|
||||
for (const snapshot of snapshots.leaves) {
|
||||
const tabId = findTerminalTabIdForLeaf(session, snapshot.leafId)
|
||||
const layout = tabId ? session?.terminalLayoutsByTabId?.[tabId] : undefined
|
||||
if (
|
||||
hasLiveLease(snapshot) &&
|
||||
layout &&
|
||||
layout.ptyIdsByLeafId?.[snapshot.leafId] === undefined
|
||||
) {
|
||||
layout.ptyIdsByLeafId = {
|
||||
...layout.ptyIdsByLeafId,
|
||||
[snapshot.leafId]: snapshot.ptyId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
markSshRemotePtyLeases(targetId: string, state: SshRemotePtyLease['state']): void {
|
||||
if (this.updateSshRemotePtyLeaseStates(targetId, state)) {
|
||||
this.flush()
|
||||
@@ -7741,7 +7408,6 @@ export class Store {
|
||||
state: SshRemotePtyLease['state']
|
||||
): Promise<void> {
|
||||
if (this.updateSshRemotePtyLeaseStates(targetId, state)) {
|
||||
this.scheduleSave()
|
||||
await this.flushDurableStateOrThrowAsync()
|
||||
}
|
||||
}
|
||||
@@ -7754,30 +7420,10 @@ export class Store {
|
||||
ptyIds.map((ptyId) => this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId))
|
||||
)
|
||||
if (this.updateSshRemotePtyLeaseStates(targetId, 'attached', relayPtyIds)) {
|
||||
this.scheduleSave()
|
||||
await this.flushDurableStateOrThrowAsync()
|
||||
}
|
||||
}
|
||||
|
||||
async markSshRemotePtyLeasesTerminatedAsync(
|
||||
targetId: string,
|
||||
ptyIds: readonly string[]
|
||||
): Promise<void> {
|
||||
const relayPtyIds = new Set(
|
||||
ptyIds.map((ptyId) => this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId))
|
||||
)
|
||||
if (this.updateSshRemotePtyLeaseStates(targetId, 'terminated', relayPtyIds)) {
|
||||
this.scheduleSave()
|
||||
try {
|
||||
await this.flushDurableStateOrThrowAsync(false)
|
||||
} catch (error) {
|
||||
// Keep the proven exit retryable after a transient persistence failure.
|
||||
this.scheduleSave()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private updateSshRemotePtyLeaseStates(
|
||||
targetId: string,
|
||||
state: SshRemotePtyLease['state'],
|
||||
@@ -7796,13 +7442,8 @@ export class Store {
|
||||
continue
|
||||
}
|
||||
if (state === 'detached' && lease.state !== 'attached') {
|
||||
if (lease.state === 'expired') {
|
||||
// A later detach fences rollback from restoring attached ownership.
|
||||
this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
}
|
||||
continue
|
||||
}
|
||||
this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
if (lease.state !== state) {
|
||||
lease.state = state
|
||||
lease.updatedAt = now
|
||||
@@ -7831,7 +7472,6 @@ export class Store {
|
||||
if (!lease) {
|
||||
return
|
||||
}
|
||||
this.advanceSshRemotePtyLeaseMutationVersion(lease)
|
||||
const shouldClearBindings = state === 'terminated' || state === 'expired'
|
||||
if (lease.state === state) {
|
||||
if (shouldClearBindings && this.clearSshRemotePtyBindingsForLeases(targetId, [lease])) {
|
||||
@@ -7885,27 +7525,19 @@ export class Store {
|
||||
this.clearSshRemotePtyBindingsForLeases(targetId, leases ?? [])
|
||||
}
|
||||
|
||||
/** `arbitratedFrom` names the plane a supersession DECISION was made from. A decision reached by
|
||||
* reading one plane may only mutate that plane: the other plane's binding never got a vote, and
|
||||
* deleting it strands a shell its owner can still be using. An explicit expiry or termination is
|
||||
* plane-agnostic — the pty is gone for everyone — so those callers pass nothing. */
|
||||
private clearSshRemotePtyBindingsForLeases(
|
||||
targetId: string,
|
||||
leases: SshRemotePtyLease[],
|
||||
arbitratedFrom?: 'local'
|
||||
leases: SshRemotePtyLease[]
|
||||
): boolean {
|
||||
if (!leases?.length) {
|
||||
return false
|
||||
}
|
||||
let changed = false
|
||||
const sessions = new Set(
|
||||
(arbitratedFrom === 'local'
|
||||
? [this.state.workspaceSession]
|
||||
: [
|
||||
this.state.workspaceSession,
|
||||
this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)]
|
||||
]
|
||||
).filter((session): session is WorkspaceSessionState => Boolean(session))
|
||||
[
|
||||
this.state.workspaceSession,
|
||||
this.state.workspaceSessionsByHostId?.[toSshExecutionHostId(targetId)]
|
||||
].filter((session): session is WorkspaceSessionState => Boolean(session))
|
||||
)
|
||||
for (const session of sessions) {
|
||||
for (const [worktreeId, tabs] of Object.entries(session.tabsByWorktree ?? {})) {
|
||||
@@ -8014,11 +7646,10 @@ export class Store {
|
||||
|
||||
// Async twin of flushOrThrow: durable state only. Active-view and GitHub sidecars are
|
||||
// quit/startup work and must not be snapshotted on the live SSH establish/reconnect path.
|
||||
private async flushDurableStateOrThrowAsync(drainToStableGeneration = true): Promise<void> {
|
||||
private async flushDurableStateOrThrowAsync(): Promise<void> {
|
||||
if (this.writesFrozen || this.quitFlushStarted) {
|
||||
throw new Error('Cannot flush while persistence is finalized')
|
||||
}
|
||||
const requiredDurableGeneration = this.writeGeneration
|
||||
for (;;) {
|
||||
if (this.writeTimer) {
|
||||
clearTimeout(this.writeTimer)
|
||||
@@ -8027,12 +7658,6 @@ export class Store {
|
||||
this.firstPendingSaveAt = null
|
||||
const generation = this.writeGeneration
|
||||
await this.enqueueWrite()
|
||||
if (
|
||||
!drainToStableGeneration &&
|
||||
this.lastDurableWriteGeneration >= requiredDurableGeneration
|
||||
) {
|
||||
break
|
||||
}
|
||||
if (generation === this.writeGeneration) {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -123,8 +123,7 @@ export type IPtyProvider = {
|
||||
/** Whether fresh structured creates can replay one spawn across a lost relay response. */
|
||||
supportsAgentSessionCreateOperations?: (options?: PtyProbeOptions) => boolean | Promise<boolean>
|
||||
attach(id: string): Promise<Pick<PtySpawnResult, 'providerSequence'> | void>
|
||||
/** Three-valued: null means the provider cannot see this id right now — absence is unproven, never dead. */
|
||||
hasPty?: (id: string) => boolean | null
|
||||
hasPty?: (id: string) => boolean
|
||||
/** Exact provider readback: false only when the provider answered that the PTY is absent. */
|
||||
probePtyLiveness?: (id: string) => Promise<boolean | null>
|
||||
write(id: string, data: string): void
|
||||
|
||||
@@ -1,30 +1,12 @@
|
||||
export {
|
||||
SSH_SESSION_EXPIRED_ERROR,
|
||||
SSH_PTY_IDENTITY_MISMATCH_ERROR,
|
||||
SSH_PTY_EXITED_ERROR,
|
||||
SSH_SOURCE_RESTORE_REQUIRED_ERROR
|
||||
} from '../../shared/ssh-pty-failure-tokens'
|
||||
import {
|
||||
SSH_SOURCE_RESTORE_REQUIRED_ERROR,
|
||||
isSshPtyExitedMessage,
|
||||
isSshPtyIdentityMismatchMessage
|
||||
} from '../../shared/ssh-pty-failure-tokens'
|
||||
|
||||
export function isSshSourceRestoreRequiredError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return message.includes(SSH_SOURCE_RESTORE_REQUIRED_ERROR)
|
||||
}
|
||||
export const SSH_SESSION_EXPIRED_ERROR = 'SSH_SESSION_EXPIRED'
|
||||
export const SSH_PTY_IDENTITY_MISMATCH_ERROR = 'SSH_PTY_IDENTITY_MISMATCH'
|
||||
|
||||
export function isSshPtyNotFoundError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return /PTY ".+" not found/i.test(message)
|
||||
}
|
||||
|
||||
/** The relay watched this shell exit — first-hand proof, unlike a bare not-found. */
|
||||
export function isSshPtyExitedError(error: unknown): boolean {
|
||||
return isSshPtyExitedMessage(error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
|
||||
export function isSshPtyIdentityMismatchError(error: unknown): boolean {
|
||||
return isSshPtyIdentityMismatchMessage(error instanceof Error ? error.message : String(error))
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return message.includes(SSH_PTY_IDENTITY_MISMATCH_ERROR) || /identity mismatch/i.test(message)
|
||||
}
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
/**
|
||||
* An identity mismatch means the relay FOUND the PTY and its recorded pane
|
||||
* differs — the shell is running. Publishing that as expiry makes the renderer
|
||||
* clear the binding and cold-restore with agent resume, which is a second agent
|
||||
* on one transcript while the first keeps running.
|
||||
*
|
||||
* Reachable today from a shipped gesture: detaching a pane into a new tab
|
||||
* changes tabId, the relay still holds the tabId frozen at spawn, and the
|
||||
* reattach mismatches.
|
||||
*
|
||||
* Pins the PRODUCER and the CONSUMERS together. The two guards this program
|
||||
* shipped before both passed while sitting off the route production takes, so
|
||||
* asserting the classifier alone is not enough — the destructive token has to
|
||||
* be absent from what the reattach actually throws.
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { reattachSshPtySession } from './ssh-pty-session-reattach'
|
||||
import {
|
||||
SSH_SESSION_EXPIRED_ERROR,
|
||||
SSH_PTY_IDENTITY_MISMATCH_ERROR,
|
||||
formatPtyExitedError
|
||||
} from '../../shared/ssh-pty-failure-tokens'
|
||||
|
||||
/** The renderer converts a failure into `sessionExpired: true` on these tokens
|
||||
* and then respawns on the flag alone, never consulting the classifier.
|
||||
* Mirrors pty-transport.ts. */
|
||||
const SSH_PTY_CONNECTION_MISMATCH_MARKER = 'belongs to SSH connection'
|
||||
function transportWouldReportSessionExpired(message: string): boolean {
|
||||
return (
|
||||
message.includes(SSH_SESSION_EXPIRED_ERROR) ||
|
||||
message.includes(SSH_PTY_CONNECTION_MISMATCH_MARKER)
|
||||
)
|
||||
}
|
||||
|
||||
const CONNECTION_ID = 'ssh-target-1'
|
||||
const RELAY_PTY_ID = 'pty-7'
|
||||
|
||||
function muxThatFailsAttachWith(message: string): { mux: unknown } {
|
||||
return {
|
||||
mux: {
|
||||
request: vi.fn().mockRejectedValue(new Error(message)),
|
||||
notify: vi.fn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function reattachError(
|
||||
attachFailure: string,
|
||||
expectedIncarnationId?: string
|
||||
): Promise<Error> {
|
||||
const { mux } = muxThatFailsAttachWith(attachFailure)
|
||||
try {
|
||||
await reattachSshPtySession({
|
||||
mux: mux as never,
|
||||
connectionId: CONNECTION_ID,
|
||||
sessionId: RELAY_PTY_ID,
|
||||
options: {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
paneKey: 'tab-new:leaf-1',
|
||||
tabId: 'tab-new',
|
||||
...(expectedIncarnationId ? { expectedIncarnationId } : {})
|
||||
} as never
|
||||
})
|
||||
} catch (error) {
|
||||
return error as Error
|
||||
}
|
||||
throw new Error('reattach unexpectedly succeeded')
|
||||
}
|
||||
|
||||
describe('an identity mismatch is not a death', () => {
|
||||
it('does not publish the destructive expiry token', async () => {
|
||||
const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`)
|
||||
|
||||
expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
})
|
||||
|
||||
// The renderer keys off this token; its own clauses live beside the
|
||||
// classifier, in reattach-failure-classification.test.ts.
|
||||
it('marks the failure with the shared mismatch token', async () => {
|
||||
const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`)
|
||||
|
||||
expect(error.message).toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR)
|
||||
})
|
||||
|
||||
// The flag path bypasses the classifier entirely, so it needs its own clause.
|
||||
it('does not trip the renderer transport into sessionExpired', async () => {
|
||||
const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found (identity mismatch)`)
|
||||
|
||||
expect(transportWouldReportSessionExpired(error.message)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('only an exit the relay watched is a death', () => {
|
||||
// Clause-selectivity: a fix that silences real expiry would strand panes on a
|
||||
// shell that truly went away, so the contrast has to hold.
|
||||
it('publishes expiry when the relay reports the exit it observed', async () => {
|
||||
const error = await reattachError(
|
||||
formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-host-7'),
|
||||
'inc-host-7'
|
||||
)
|
||||
|
||||
expect(error.message).toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
expect(error.message).not.toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR)
|
||||
expect(transportWouldReportSessionExpired(error.message)).toBe(true)
|
||||
})
|
||||
|
||||
// INVERTED. This used to require a bare absence to publish expiry. That is the defect: the relay
|
||||
// we asked cannot hand the id back, which proves an exit ONLY if it is the relay that minted it.
|
||||
// A replaced relay answers exactly this for shells still running under its predecessor, so
|
||||
// expiry there cleared ownership and resumed the agent a second time onto a live shell. The
|
||||
// death that is real is still detected — by the clause above, from an exit the relay watched.
|
||||
// Enforcement cannot live only on the host: the host is the party whose answer is in question,
|
||||
// and versions differ. A proof we cannot tie to the shell we asked about is not proof.
|
||||
it('does not publish expiry for an exit that names a different shell', async () => {
|
||||
const error = await reattachError(
|
||||
formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-some-other-shell'),
|
||||
'inc-host-7'
|
||||
)
|
||||
|
||||
expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
expect(transportWouldReportSessionExpired(error.message)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not publish expiry for an exit that names a different PTY id', async () => {
|
||||
const error = await reattachError(
|
||||
formatPtyExitedError('pty-some-other-shell', 0, 'inc-host-7'),
|
||||
'inc-host-7'
|
||||
)
|
||||
|
||||
expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
expect(transportWouldReportSessionExpired(error.message)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not publish expiry for an exit when the pane knows no incarnation', async () => {
|
||||
const error = await reattachError(formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-host-7'))
|
||||
|
||||
expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
})
|
||||
|
||||
it('does not publish expiry when the relay merely has no such PTY', async () => {
|
||||
const error = await reattachError(`PTY "${RELAY_PTY_ID}" not found`)
|
||||
|
||||
expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
expect(transportWouldReportSessionExpired(error.message)).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves an unrelated failure untouched rather than guessing', async () => {
|
||||
const error = await reattachError('ECONNRESET while writing to the relay')
|
||||
|
||||
expect(error.message).not.toContain(SSH_SESSION_EXPIRED_ERROR)
|
||||
expect(error.message).not.toContain(SSH_PTY_IDENTITY_MISMATCH_ERROR)
|
||||
})
|
||||
})
|
||||
|
||||
// New relays prioritize incarnation, so the pane fields do not reject a moved pane. Old relays
|
||||
// ignore the additive incarnation field and still need their weaker pane fence.
|
||||
describe('reattach preserves the old-relay pane fence', () => {
|
||||
async function attachParams(
|
||||
extraOptions: Record<string, unknown> = {}
|
||||
): Promise<Record<string, unknown>> {
|
||||
const request = vi.fn().mockRejectedValue(new Error('boom'))
|
||||
try {
|
||||
await reattachSshPtySession({
|
||||
mux: { request, notify: vi.fn() } as never,
|
||||
connectionId: CONNECTION_ID,
|
||||
sessionId: RELAY_PTY_ID,
|
||||
options: {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
paneKey: 'tab-new:leaf-1',
|
||||
tabId: 'tab-new',
|
||||
env: { ORCA_PANE_KEY: 'tab-old:leaf-1', ORCA_TAB_ID: 'tab-old' },
|
||||
...extraOptions
|
||||
} as never
|
||||
})
|
||||
} catch {
|
||||
// the attach failure is not what this clause is about
|
||||
}
|
||||
const call = request.mock.calls.at(0)
|
||||
return (call?.[1] ?? call?.[0] ?? {}) as Record<string, unknown>
|
||||
}
|
||||
|
||||
it('sends the current pane key for relays that do not understand incarnation', async () => {
|
||||
expect(await attachParams()).toMatchObject({ expectedPaneKey: 'tab-new:leaf-1' })
|
||||
})
|
||||
|
||||
it('sends the current tab id for relays that do not understand incarnation', async () => {
|
||||
expect(await attachParams()).toMatchObject({ expectedTabId: 'tab-new' })
|
||||
})
|
||||
|
||||
// The producer pin for the relay's incarnation guard. The guard is only worth having if the
|
||||
// expectation actually leaves the client, and nothing else here would notice if it stopped:
|
||||
// the relay stays permissive on an absent field, so a silent regression reads as "all green".
|
||||
it('sends the expected incarnation when the host attested one', async () => {
|
||||
expect(await attachParams({ expectedIncarnationId: 'inc-host-7' })).toMatchObject({
|
||||
expectedIncarnationId: 'inc-host-7',
|
||||
expectedPaneKey: 'tab-new:leaf-1',
|
||||
expectedTabId: 'tab-new'
|
||||
})
|
||||
})
|
||||
|
||||
// A stand-in minted locally when the host reported none. It is first-write-wins and is dropped
|
||||
// when provider state resets, so the same live shell can present a different one after a
|
||||
// reconnect — sending it would make the relay refuse the pane its own shell.
|
||||
it('sends no expected incarnation when the value was synthesized locally', async () => {
|
||||
expect(await attachParams({ expectedIncarnationId: 'legacy:23:0:pty-7' })).not.toHaveProperty(
|
||||
'expectedIncarnationId'
|
||||
)
|
||||
})
|
||||
|
||||
it('sends no expected incarnation when there is none to send', async () => {
|
||||
expect(await attachParams()).not.toHaveProperty('expectedIncarnationId')
|
||||
})
|
||||
})
|
||||
@@ -316,8 +316,7 @@ describe('SSH fresh agent-session create operations', () => {
|
||||
)
|
||||
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
// Not `false`: the provider has never listed this host, so absence is unknown, not proven.
|
||||
expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).not.toBe(true)
|
||||
expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(false)
|
||||
const shutdownRequest = await waitForRequest(transport, 'pty.shutdown')
|
||||
transport.deliver(responseFrame(shutdownRequest.id as number, null, 4))
|
||||
const cancelRequest = await waitForRequest(transport, 'pty.cancelDelivery')
|
||||
|
||||
@@ -170,30 +170,3 @@ it('returns a provisional source activation lease to reconnect authority', async
|
||||
deliveryToken: 'token-reconnect'
|
||||
})
|
||||
})
|
||||
|
||||
it('fences a mismatched attach response before installing its source activation', async () => {
|
||||
const response = {
|
||||
incarnationId: 'incarnation-other',
|
||||
sourceActivation: sourceActivation('incarnation-other')
|
||||
}
|
||||
const mux = {
|
||||
request: vi.fn(async (method: string, _params, options) => {
|
||||
if (method !== 'pty.attach') {
|
||||
return undefined
|
||||
}
|
||||
options?.beforeResolve?.(response)
|
||||
return response
|
||||
}),
|
||||
notify: vi.fn(),
|
||||
onNotification: vi.fn(),
|
||||
dispose: vi.fn(),
|
||||
isDisposed: vi.fn().mockReturnValue(false)
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
|
||||
await expect(
|
||||
provider.attachForReconnect('ssh:conn-1@@pty-1', undefined, 'incarnation-expected')
|
||||
).rejects.toThrow('SSH_PTY_IDENTITY_MISMATCH')
|
||||
|
||||
expect(mux.request).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
type SshPtyReceivingActivationLease
|
||||
} from './ssh-pty-notification-routing'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
import { SYNTHESIZED_PTY_INCARNATION_PREFIX } from '../../shared/pty-incarnation'
|
||||
|
||||
export class SshPtyProviderOutputState {
|
||||
private readonly dataListeners = new Set<SshPtyDataCallback>()
|
||||
@@ -133,7 +132,7 @@ export class SshPtyProviderOutputState {
|
||||
this.rememberPtyIncarnation(relayPtyId, incarnationId)
|
||||
let resolved = this.incarnationByRelayPtyId.get(relayPtyId)
|
||||
if (!resolved) {
|
||||
resolved = `${SYNTHESIZED_PTY_INCARNATION_PREFIX}${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}`
|
||||
resolved = `legacy:${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}`
|
||||
this.incarnationByRelayPtyId.set(relayPtyId, resolved)
|
||||
}
|
||||
return resolved
|
||||
|
||||
@@ -29,17 +29,6 @@ describe('SshPtyProvider process listings and events', () => {
|
||||
provider = new SshPtyProvider('conn-1', mux as never)
|
||||
})
|
||||
|
||||
// Why: a reconnect builds a fresh provider with an empty set, so a miss before the
|
||||
// first completed listing is ignorance about the host, not a dead PTY.
|
||||
it('answers unknown liveness until a completed listing makes absence provable', async () => {
|
||||
expect(provider.hasPty(scopedPty1)).toBe(null)
|
||||
|
||||
mux.request.mockResolvedValueOnce([])
|
||||
await provider.listProcesses()
|
||||
|
||||
expect(provider.hasPty(scopedPty1)).toBe(false)
|
||||
})
|
||||
|
||||
it('scopes process listings and bounds the relay RPC by the teardown deadline', async () => {
|
||||
const processes = [{ id: 'pty-1', cwd: '/home', title: 'zsh', worktreeId: 'repo::/home' }]
|
||||
mux.request.mockResolvedValue(processes)
|
||||
|
||||
@@ -1,28 +1,8 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
SSH_PTY_IDENTITY_MISMATCH_ERROR,
|
||||
SSH_SESSION_EXPIRED_ERROR,
|
||||
SSH_SOURCE_RESTORE_REQUIRED_ERROR
|
||||
} from './ssh-pty-errors'
|
||||
import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors'
|
||||
import { SshPtyProvider } from './ssh-pty-provider'
|
||||
|
||||
describe('SSH PTY provider session reattach incarnation', () => {
|
||||
it.each([
|
||||
['an old-relay response without an incarnation', {}],
|
||||
['a response for a different incarnation', { incarnationId: 'incarnation-other' }]
|
||||
])('rejects %s', async (_label, response) => {
|
||||
const mux = {
|
||||
request: vi.fn().mockResolvedValue(response),
|
||||
notify: vi.fn(),
|
||||
onNotification: vi.fn().mockReturnValue(vi.fn())
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
|
||||
await expect(
|
||||
provider.attachForReconnect('ssh:conn-1@@pty-old', undefined, 'incarnation-expected')
|
||||
).rejects.toThrow(SSH_PTY_IDENTITY_MISMATCH_ERROR)
|
||||
})
|
||||
|
||||
it('remembers the authoritative incarnation before a legacy exit arrives', async () => {
|
||||
let notify: ((method: string, params: Record<string, unknown>) => void) | undefined
|
||||
const mux = {
|
||||
@@ -64,10 +44,8 @@ describe('SSH PTY provider session reattach incarnation', () => {
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
|
||||
// Fails closed, but NOT as expiry: the shell is still running, and callers
|
||||
// respawn on expiry — which duplicate-resumed the live agent session.
|
||||
const spawn = provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })
|
||||
await expect(spawn).rejects.toThrow(`${SSH_SOURCE_RESTORE_REQUIRED_ERROR}: pty-old`)
|
||||
await expect(spawn).rejects.not.toThrow(SSH_SESSION_EXPIRED_ERROR)
|
||||
await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow(
|
||||
`${SSH_SESSION_EXPIRED_ERROR}: pty-old`
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { describe, expect, it, vi, beforeEach } from 'vitest'
|
||||
import { formatPtyExitedError } from '../../shared/ssh-pty-failure-tokens'
|
||||
import { SshPtyProvider } from './ssh-pty-provider'
|
||||
import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env'
|
||||
import { PTY_STARTUP_INGRESS_VERSION } from '../../shared/pty-startup-ingress'
|
||||
@@ -585,8 +584,7 @@ describe('SshPtyProvider', () => {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true,
|
||||
exitProofSupported: true
|
||||
suppressReplayNotification: true
|
||||
},
|
||||
sourceActivationRequestOptions
|
||||
)
|
||||
@@ -632,8 +630,7 @@ describe('SshPtyProvider', () => {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true,
|
||||
exitProofSupported: true
|
||||
suppressReplayNotification: true
|
||||
})
|
||||
expect(result).toEqual({
|
||||
id: 'ssh:conn-1@@pty-old',
|
||||
@@ -642,8 +639,7 @@ describe('SshPtyProvider', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// New relays prioritize incarnation; old relays ignore that field and retain this weaker fence.
|
||||
it('preserves pane identity for relays that predate incarnation fencing', async () => {
|
||||
it('reattaches with explicit pane identity when hook env was stripped', async () => {
|
||||
mux.request.mockResolvedValue({ replay: 'buffered-output' })
|
||||
|
||||
await provider.spawn({
|
||||
@@ -659,29 +655,17 @@ describe('SshPtyProvider', () => {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true,
|
||||
exitProofSupported: true,
|
||||
expectedPaneKey: 'tab-a:leaf-a',
|
||||
expectedTabId: 'tab-a'
|
||||
})
|
||||
})
|
||||
|
||||
it('does not fresh-spawn over a reattach whose shell the relay saw exit', async () => {
|
||||
// INVERTED to drive the exit the relay OBSERVED. A bare not-found no longer becomes expiry —
|
||||
// it proves nothing when the answering relay may be a replacement — so the observed exit is
|
||||
// now what carries this through to expiry. The property under test is unchanged: a failed
|
||||
// reattach throws instead of quietly spawning a second shell over the pane.
|
||||
mux.request.mockRejectedValueOnce(
|
||||
new Error(formatPtyExitedError('pty-old', 0, 'inc-host-old'))
|
||||
)
|
||||
it('does not fresh-spawn over an expired reattach session', async () => {
|
||||
mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found'))
|
||||
|
||||
await expect(
|
||||
provider.spawn({
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
sessionId: 'pty-old',
|
||||
expectedIncarnationId: 'inc-host-old'
|
||||
})
|
||||
).rejects.toThrow('SSH_SESSION_EXPIRED: pty-old')
|
||||
await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow(
|
||||
'SSH_SESSION_EXPIRED: pty-old'
|
||||
)
|
||||
|
||||
expect(mux.request).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
@@ -690,27 +674,13 @@ describe('SshPtyProvider', () => {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true,
|
||||
exitProofSupported: true,
|
||||
expectedIncarnationId: 'inc-host-old'
|
||||
suppressReplayNotification: true
|
||||
},
|
||||
sourceActivationRequestOptions
|
||||
)
|
||||
expect(mux.request).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
// The sibling case, and the one the change is for: an id the relay simply does not know may be
|
||||
// a shell still running under a relay this one replaced. It must still refuse to fresh-spawn,
|
||||
// and must NOT be dressed up as expiry, which is what authorized the replacement.
|
||||
it('does not fresh-spawn, nor claim expiry, when the relay merely has no such PTY', async () => {
|
||||
mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found'))
|
||||
|
||||
await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow(
|
||||
'PTY "pty-old" not found'
|
||||
)
|
||||
expect(mux.request).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('preserves transient reattach failures for retry handling', async () => {
|
||||
mux.request.mockRejectedValueOnce(new Error('SSH connection lost, reconnecting...'))
|
||||
|
||||
@@ -744,8 +714,7 @@ describe('SshPtyProvider', () => {
|
||||
'pty.attach',
|
||||
{
|
||||
id: 'pty-1',
|
||||
suppressReplayNotification: true,
|
||||
exitProofSupported: true
|
||||
suppressReplayNotification: true
|
||||
},
|
||||
expect.objectContaining({
|
||||
timeoutMs: 10_000,
|
||||
@@ -770,6 +739,28 @@ describe('SshPtyProvider', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('attachForReconnect forwards expected identity when provided', async () => {
|
||||
await provider.attachForReconnect(scopedPty1, {
|
||||
paneKey: 'tab-a:leaf-a',
|
||||
tabId: 'tab-a'
|
||||
})
|
||||
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.attach',
|
||||
{
|
||||
id: 'pty-1',
|
||||
suppressReplayNotification: true,
|
||||
expectedPaneKey: 'tab-a:leaf-a',
|
||||
expectedTabId: 'tab-a'
|
||||
},
|
||||
expect.objectContaining({
|
||||
timeoutMs: 10_000,
|
||||
beforeResolve: expect.any(Function)
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('write sends pty.data notification', () => {
|
||||
provider.write(scopedPty1, 'hello')
|
||||
expect(mux.notify).toHaveBeenCalledWith('pty.data', { id: 'pty-1', data: 'hello' })
|
||||
|
||||
@@ -13,7 +13,6 @@ import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state'
|
||||
import { spawnFreshSshPty } from './ssh-agent-session-create-operation'
|
||||
import { mapSshPtyProcessList } from './ssh-agent-session-process-list'
|
||||
import {
|
||||
buildSshPtyReconnectAttachParams,
|
||||
requestSshPtyAttach,
|
||||
reattachSshPtySessionWithExitFence,
|
||||
type PtySourceRecoveryRequest,
|
||||
@@ -23,7 +22,7 @@ import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request'
|
||||
import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
|
||||
import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities'
|
||||
import type { PtyProcessInspection } from './pty-process-inspection'
|
||||
import { SSH_SOURCE_RESTORE_REQUIRED_ERROR } from './ssh-pty-errors'
|
||||
import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors'
|
||||
|
||||
// Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch.
|
||||
function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined {
|
||||
@@ -35,7 +34,6 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
private mux: SshChannelMultiplexer
|
||||
private connectionId: string
|
||||
private livePtyIds = new Set<string>()
|
||||
private listedOnce = false
|
||||
readonly getAppliedSize: NonNullable<IPtyProvider['getAppliedSize']>
|
||||
private readonly agentSessionCapabilities: SshAgentSessionCapabilities
|
||||
private spawnExitRaces = new SshPtySpawnExitRaceTracker()
|
||||
@@ -65,7 +63,6 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
dispose(): void {
|
||||
this.outputState.dispose()
|
||||
this.livePtyIds.clear()
|
||||
this.listedOnce = false
|
||||
}
|
||||
|
||||
getConnectionId = (): string => this.connectionId
|
||||
@@ -104,11 +101,8 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId)
|
||||
})
|
||||
if (result.sourceRecovery?.status === 'restoreRequired') {
|
||||
// Why not SSH_SESSION_EXPIRED: the shell is still running, only its
|
||||
// output source needs re-establishing. Reporting expiry made the pane
|
||||
// respawn and resume the same agent session twice into one transcript.
|
||||
throw new Error(
|
||||
`${SSH_SOURCE_RESTORE_REQUIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}`
|
||||
`${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}`
|
||||
)
|
||||
}
|
||||
this.livePtyIds.add(result.id)
|
||||
@@ -183,16 +177,20 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
|
||||
async attachForReconnect(
|
||||
id: string,
|
||||
sourceRecovery?: PtySourceRecoveryRequest,
|
||||
expectedIncarnationId?: string,
|
||||
legacyExpectedIdentity?: { paneKey?: string; tabId?: string }
|
||||
expected?: { paneKey?: string; tabId?: string },
|
||||
sourceRecovery?: PtySourceRecoveryRequest
|
||||
): Promise<SshPtyAttachResult> {
|
||||
const params = buildSshPtyReconnectAttachParams({
|
||||
// Why: reconnect owns replay delivery so stale/duplicate attach results can
|
||||
// be filtered before they reach the renderer. The expected identity lets the
|
||||
// relay reject a cross-generation id collision instead of reattaching this
|
||||
// lease to a different pane's freshly spawned PTY.
|
||||
const params = {
|
||||
id: this.toRelayPtyId(id),
|
||||
suppressReplayNotification: true,
|
||||
...(sourceRecovery ? { sourceRecovery } : {}),
|
||||
...(expectedIncarnationId ? { expectedIncarnationId } : {}),
|
||||
...(legacyExpectedIdentity ? { legacyExpectedIdentity } : {})
|
||||
})
|
||||
...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}),
|
||||
...(expected?.tabId ? { expectedTabId: expected.tabId } : {})
|
||||
}
|
||||
const relayPtyId = this.toRelayPtyId(id)
|
||||
return await requestSshPtyAttach({
|
||||
mux: this.mux,
|
||||
@@ -298,14 +296,11 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
const relayPtyId = this.toRelayPtyId(process.id)
|
||||
this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId)
|
||||
}
|
||||
this.listedOnce = true
|
||||
return processes
|
||||
}
|
||||
|
||||
hasPty(id: string): boolean | null {
|
||||
// Why null before a completed listing: a reconnect builds a new provider with an
|
||||
// empty set, so a miss there is ignorance about the host, not a dead PTY.
|
||||
return this.livePtyIds.has(id) ? true : this.listedOnce ? false : null
|
||||
hasPty(id: string): boolean {
|
||||
return this.livePtyIds.has(id)
|
||||
}
|
||||
|
||||
async getDefaultShell(): Promise<string> {
|
||||
|
||||
@@ -1,16 +1,11 @@
|
||||
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import {
|
||||
isPtyIncarnationId,
|
||||
isRelayAttestedPtyIncarnationId,
|
||||
type PtyIncarnationId
|
||||
} from '../../shared/pty-incarnation'
|
||||
import { isPtyIncarnationId, type PtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import {
|
||||
SSH_PTY_IDENTITY_MISMATCH_ERROR,
|
||||
SSH_SESSION_EXPIRED_ERROR,
|
||||
isSshPtyExitedError,
|
||||
isSshPtyIdentityMismatchError
|
||||
isSshPtyIdentityMismatchError,
|
||||
isSshPtyNotFoundError
|
||||
} from './ssh-pty-errors'
|
||||
import { parseMatchingPtyExitedError } from '../../shared/ssh-pty-failure-tokens'
|
||||
import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id'
|
||||
import type { PtySpawnOptions, PtySpawnResult } from './types'
|
||||
import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
|
||||
@@ -32,29 +27,6 @@ export type SshPtyAttachResult = {
|
||||
sourceActivationLease?: SshPtyReceivingActivationLease
|
||||
}
|
||||
|
||||
export function buildSshPtyReconnectAttachParams(args: {
|
||||
id: string
|
||||
sourceRecovery?: PtySourceRecoveryRequest
|
||||
expectedIncarnationId?: string
|
||||
legacyExpectedIdentity?: { paneKey?: string; tabId?: string }
|
||||
}): Record<string, unknown> {
|
||||
return {
|
||||
id: args.id,
|
||||
suppressReplayNotification: true,
|
||||
exitProofSupported: true,
|
||||
...(isRelayAttestedPtyIncarnationId(args.expectedIncarnationId)
|
||||
? { expectedIncarnationId: args.expectedIncarnationId }
|
||||
: {}),
|
||||
...(args.legacyExpectedIdentity?.paneKey
|
||||
? { expectedPaneKey: args.legacyExpectedIdentity.paneKey }
|
||||
: {}),
|
||||
...(args.legacyExpectedIdentity?.tabId
|
||||
? { expectedTabId: args.legacyExpectedIdentity.tabId }
|
||||
: {}),
|
||||
...(args.sourceRecovery ? { sourceRecovery: args.sourceRecovery } : {})
|
||||
}
|
||||
}
|
||||
|
||||
type SshPtyReattachResult = PtySpawnResult & {
|
||||
sourceRecovery?: PtySourceRecoveryResult
|
||||
sourceActivationLease?: SshPtyReceivingActivationLease
|
||||
@@ -113,18 +85,6 @@ export async function requestSshPtyAttach(args: {
|
||||
rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void
|
||||
}): Promise<SshPtyAttachResult> {
|
||||
let activationLease: SshPtyReceivingActivationLease | undefined
|
||||
const parseResult = (value: unknown): SshPtyAttachResult => {
|
||||
const result = parseSshPtyAttachResult(value)
|
||||
const expectedIncarnationId = args.params.expectedIncarnationId
|
||||
// Old relays ignore the request fence, so their response must prove the same shell.
|
||||
if (
|
||||
isRelayAttestedPtyIncarnationId(expectedIncarnationId) &&
|
||||
result.incarnationId !== expectedIncarnationId
|
||||
) {
|
||||
throw new Error(`${SSH_PTY_IDENTITY_MISMATCH_ERROR}: ${args.relayPtyId}`)
|
||||
}
|
||||
return result
|
||||
}
|
||||
const installFromResult = (result: SshPtyAttachResult): void => {
|
||||
if (!activationLease && result.sourceActivation && args.installSourceActivation) {
|
||||
activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation)
|
||||
@@ -133,9 +93,9 @@ export async function requestSshPtyAttach(args: {
|
||||
try {
|
||||
const rawResult = await args.mux.request('pty.attach', args.params, {
|
||||
...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }),
|
||||
beforeResolve: (value) => installFromResult(parseResult(value))
|
||||
beforeResolve: (value) => installFromResult(parseSshPtyAttachResult(value))
|
||||
})
|
||||
const result = parseResult(rawResult)
|
||||
const result = parseSshPtyAttachResult(rawResult)
|
||||
installFromResult(result)
|
||||
args.rememberPtyIncarnation?.(args.relayPtyId, result.incarnationId)
|
||||
if (args.commitSourceActivation) {
|
||||
@@ -225,6 +185,7 @@ export async function reattachSshPtySession(args: {
|
||||
const relaySessionId = toRelaySshPtyId(args.connectionId, args.sessionId)
|
||||
console.warn(`[ssh-pty] spawn() called with sessionId=${args.sessionId}, attempting pty.attach`)
|
||||
try {
|
||||
// Why: expected pane identity prevents a reused relay id from attaching the wrong shell.
|
||||
const expectedPaneKey = args.options.paneKey ?? args.options.env?.ORCA_PANE_KEY
|
||||
const expectedTabId = args.options.tabId ?? args.options.env?.ORCA_TAB_ID
|
||||
const attachResult = await requestSshPtyAttach({
|
||||
@@ -235,17 +196,6 @@ export async function reattachSshPtySession(args: {
|
||||
cols: args.options.cols,
|
||||
rows: args.options.rows,
|
||||
suppressReplayNotification: true,
|
||||
// Declares that this client can act on a proven exit. Without it a host must keep answering
|
||||
// in the older wording, because an older client reads anything else as an unknown failure
|
||||
// and leaves the pane with no shell and no way back.
|
||||
exitProofSupported: true,
|
||||
// The shell's own identity, so it survives a pane moving between tabs — unlike the pane
|
||||
// identity this replaced. Sent only when the host attested it: a locally synthesized
|
||||
// stand-in is not stable across reconnects and would refuse the pane its own shell. An
|
||||
// older relay ignores the field, so the legacy pane fence below remains its fallback.
|
||||
...(isRelayAttestedPtyIncarnationId(args.options.expectedIncarnationId)
|
||||
? { expectedIncarnationId: args.options.expectedIncarnationId }
|
||||
: {}),
|
||||
...(expectedPaneKey ? { expectedPaneKey } : {}),
|
||||
...(expectedTabId ? { expectedTabId } : {})
|
||||
},
|
||||
@@ -269,34 +219,12 @@ export async function reattachSshPtySession(args: {
|
||||
} catch (error) {
|
||||
// Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding.
|
||||
console.warn(`[ssh-pty] pty.attach FAILED for ${args.sessionId}:`, error)
|
||||
// Why: the relay reports a mismatch by saying "not found", but it found the
|
||||
// pty — comparing identity is how it knows. Publishing expiry there makes
|
||||
// the renderer respawn and resume the agent a second time onto a live shell.
|
||||
if (isSshPtyIdentityMismatchError(error)) {
|
||||
throw new Error(`${SSH_PTY_IDENTITY_MISMATCH_ERROR}: ${relaySessionId}`)
|
||||
if (isSshPtyNotFoundError(error)) {
|
||||
const mismatchMarker = isSshPtyIdentityMismatchError(error)
|
||||
? ` ${SSH_PTY_IDENTITY_MISMATCH_ERROR}`
|
||||
: ''
|
||||
throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}${mismatchMarker}`)
|
||||
}
|
||||
// The relay WATCHED this shell exit, which is the only answer that proves it is gone, so this
|
||||
// is the one route that may authorize a replacement. The proof still has to be about OUR shell:
|
||||
// the host applies that rule too, but the host is the party whose answer is in question and
|
||||
// versions differ, so a proof we cannot tie to the incarnation we asked about is not proof and
|
||||
// falls through to the disconnected pane instead of replacing a shell that may be running.
|
||||
if (isSshPtyExitedError(error)) {
|
||||
const proof = parseMatchingPtyExitedError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
relaySessionId,
|
||||
args.options.expectedIncarnationId
|
||||
)
|
||||
if (proof) {
|
||||
throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}`)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
// A bare not-found deliberately does NOT become expiry any more. It means the relay we asked
|
||||
// cannot hand the id back, which is proof of an exit only if that relay is the one that minted
|
||||
// it — and a replaced relay answers exactly this for shells still running under its
|
||||
// predecessor. Treating it as death cleared ownership and resumed the agent a second time onto
|
||||
// a live shell. Unproven now falls through to the caller, which shows the pane as
|
||||
// disconnected and lets the user decide.
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2858,9 +2858,7 @@ describe('OrcaRuntimeService', () => {
|
||||
)
|
||||
})
|
||||
|
||||
// Inverted for STA-3077 S8: the replacement grant this pinned was unreachable in production
|
||||
// (relay-native lease ptyId vs app-form runtime ptyId) and has been deleted.
|
||||
it('refuses to replace a disconnected SSH pane, even inside the lease grace window', async () => {
|
||||
it('recovers a disconnected pane through one HUB-owned replacement', async () => {
|
||||
const tabId = 'tab-recover'
|
||||
const runtime = createRuntimeWithSshLease('pty-expired', tabId)
|
||||
const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID)
|
||||
@@ -2882,8 +2880,18 @@ describe('OrcaRuntimeService', () => {
|
||||
|
||||
await expect(
|
||||
runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle)
|
||||
).rejects.toThrow('terminal_not_recoverable')
|
||||
expect(createTerminal).not.toHaveBeenCalled()
|
||||
).resolves.toMatchObject({
|
||||
handle: 'term-replacement',
|
||||
tabId,
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
})
|
||||
expect(createTerminal).toHaveBeenCalledWith(`id:${TEST_WORKTREE_ID}`, {
|
||||
tabId,
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false,
|
||||
persistHostSessionBinding: true
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects missing host panes without authoritative expired binding evidence', async () => {
|
||||
@@ -2949,9 +2957,7 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(createTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Inverted for STA-3077 S8: this was the only consumer of the recovery-dedup map, which existed
|
||||
// solely to de-duplicate the (unreachable, now deleted) replacement grant. Scenario kept.
|
||||
it('concurrent recovery requests for one disconnected pane both refuse and create nothing', async () => {
|
||||
it('deduplicates concurrent pane recovery across stale viewer handles', async () => {
|
||||
const tabId = 'tab-concurrent'
|
||||
const runtime = createRuntimeWithSshLease('pty-expired', tabId)
|
||||
const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID)
|
||||
@@ -2961,23 +2967,30 @@ describe('OrcaRuntimeService', () => {
|
||||
})
|
||||
const expiredHandle = runtime.resolveTerminalPane(paneKey, TEST_WORKTREE_ID).handle
|
||||
runtime.onPtyExit('pty-expired', 0)
|
||||
// Never settles: a resurrected grant would hang the assertion rather than quietly pass.
|
||||
const createTerminal = vi
|
||||
.spyOn(runtime, 'createTerminal')
|
||||
.mockReturnValue(new Promise<RuntimeTerminalCreate>(() => {}))
|
||||
let finishCreate!: (result: RuntimeTerminalCreate) => void
|
||||
const pendingCreate = new Promise<RuntimeTerminalCreate>((resolve) => {
|
||||
finishCreate = resolve
|
||||
})
|
||||
const createTerminal = vi.spyOn(runtime, 'createTerminal').mockReturnValue(pendingCreate)
|
||||
|
||||
const first = runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle)
|
||||
const second = runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, 'term-other-viewer')
|
||||
finishCreate({
|
||||
handle: 'term-replacement',
|
||||
tabId,
|
||||
paneKey,
|
||||
ptyId: 'pty-replacement',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: null,
|
||||
surface: 'background'
|
||||
})
|
||||
|
||||
await expect(first).rejects.toThrow('terminal_not_recoverable')
|
||||
// The second viewer's handle never mapped to this pane, so it still fails the identity guard.
|
||||
await expect(first).resolves.toEqual(expect.objectContaining({ handle: 'term-replacement' }))
|
||||
await expect(second).rejects.toThrow('terminal_not_found')
|
||||
expect(createTerminal).not.toHaveBeenCalled()
|
||||
expect(createTerminal).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
// Inverted for STA-3077 S8: it pinned retry-after-a-failed-grant, and the grant (unreachable at
|
||||
// the pty-id shapes production mints) is deleted, so there is nothing left to retry into.
|
||||
it('a repeated recovery request never accumulates a grant', async () => {
|
||||
it('clears a failed pane recovery so a later reconnect can retry', async () => {
|
||||
const tabId = 'tab-retry'
|
||||
const runtime = createRuntimeWithSshLease('pty-expired', tabId)
|
||||
const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID)
|
||||
@@ -2987,7 +3000,6 @@ describe('OrcaRuntimeService', () => {
|
||||
})
|
||||
const expiredHandle = runtime.resolveTerminalPane(paneKey, TEST_WORKTREE_ID).handle
|
||||
runtime.onPtyExit('pty-expired', 0)
|
||||
// A fully working createTerminal stays armed across both calls to prove neither one reaches it.
|
||||
const createTerminal = vi
|
||||
.spyOn(runtime, 'createTerminal')
|
||||
.mockRejectedValueOnce(new Error('relay_reconnecting'))
|
||||
@@ -3003,16 +3015,13 @@ describe('OrcaRuntimeService', () => {
|
||||
|
||||
await expect(
|
||||
runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle)
|
||||
).rejects.toThrow('terminal_not_recoverable')
|
||||
).rejects.toThrow('relay_reconnecting')
|
||||
await expect(
|
||||
runtime.recoverTerminalPane(paneKey, TEST_WORKTREE_ID, expiredHandle)
|
||||
).rejects.toThrow('terminal_not_recoverable')
|
||||
expect(createTerminal).toHaveBeenCalledTimes(0)
|
||||
).resolves.toMatchObject({ handle: 'term-retry' })
|
||||
expect(createTerminal).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
// STA-3077 S8: no longer discriminating — since the grant was deleted, an 'expired' lease refuses
|
||||
// identically (see the grace-window case above), so 'terminated' no longer selects the outcome.
|
||||
// Retained only as a regression guard: if a replacement path ever returns, this must stay a refusal.
|
||||
it('does not recover a pane whose authoritative SSH lease was terminated', async () => {
|
||||
const tabId = 'tab-terminated'
|
||||
const runtime = createRuntimeWithSshLease('pty-terminated', tabId, 'terminated')
|
||||
|
||||
@@ -2787,6 +2787,7 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
>()
|
||||
private terminalSleepGeneration = 0
|
||||
private terminalPaneRecoveryByIdentity = new Map<string, Promise<RuntimeTerminalResolvePane>>()
|
||||
// Why: idempotency map for worktree.create — a create interrupted by a mobile
|
||||
// connection migration is retried with the same clientMutationId and returns
|
||||
// the in-flight (or just-finished) operation instead of a duplicate worktree.
|
||||
@@ -6301,7 +6302,8 @@ export class OrcaRuntimeService {
|
||||
private getRecentExpiredSshLease(
|
||||
worktreeId: string,
|
||||
tabId: string,
|
||||
leafId: string | undefined
|
||||
leafId: string | undefined,
|
||||
ptyId?: string
|
||||
): ReturnType<NonNullable<RuntimeStore['getSshRemotePtyLeases']>>[number] | null {
|
||||
const now = Date.now()
|
||||
return (
|
||||
@@ -6312,6 +6314,7 @@ export class OrcaRuntimeService {
|
||||
lease.state === 'expired' &&
|
||||
lease.worktreeId === worktreeId &&
|
||||
lease.tabId === tabId &&
|
||||
(ptyId === undefined || lease.ptyId === ptyId) &&
|
||||
(leafId === undefined || lease.leafId === undefined || lease.leafId === leafId) &&
|
||||
lease.updatedAt <= now &&
|
||||
now - lease.updatedAt <= SSH_PANE_RECOVERY_GRACE_MS
|
||||
@@ -16621,17 +16624,46 @@ export class OrcaRuntimeService {
|
||||
) {
|
||||
throw new Error('terminal_not_found')
|
||||
}
|
||||
const recoveryKey = `${expectedWorktreeId}\0${paneKey}`
|
||||
const pending = this.terminalPaneRecoveryByIdentity.get(recoveryKey)
|
||||
if (pending) {
|
||||
return pending
|
||||
}
|
||||
if (pty?.connected) {
|
||||
const current = this.resolveTerminalPane(paneKey, expectedWorktreeId)
|
||||
if (expectedHandle === undefined || current.handle !== expectedHandle) {
|
||||
return current
|
||||
}
|
||||
throw new Error('terminal_not_recoverable')
|
||||
}
|
||||
// A disconnected pane is never silently replaced. The grant this used to hold could not fire:
|
||||
// it compared a relay-native lease id against an app-form runtime id (STA-3077 S6/S8, oracle in
|
||||
// ssh-pane-recovery-grant-reachability.test.ts). The pane surfaces as disconnected instead, and
|
||||
// spawning a replacement is the user's explicit choice.
|
||||
throw new Error('terminal_not_recoverable')
|
||||
if (
|
||||
!this.getRecentExpiredSshLease(expectedWorktreeId, parsed.tabId, parsed.leafId, pty.ptyId)
|
||||
) {
|
||||
// Why: an explicit close leaves a terminated lease; only relay expiry authorizes shell recreation.
|
||||
throw new Error('terminal_not_recoverable')
|
||||
}
|
||||
// Why: disconnected PTYs can reissue handles during graph cleanup; only a connected replacement satisfies the pane CAS.
|
||||
const recovery = this.createTerminal(`id:${expectedWorktreeId}`, {
|
||||
tabId: parsed.tabId,
|
||||
leafId: parsed.leafId,
|
||||
focus: false,
|
||||
// Why: the HUB renderer may publish its exited layout while recovery is in flight; persist the replacement before that stale graph can orphan it.
|
||||
persistHostSessionBinding: true
|
||||
}).then((terminal) => ({
|
||||
handle: terminal.handle,
|
||||
tabId: parsed.tabId,
|
||||
leafId: parsed.leafId,
|
||||
ptyId: terminal.ptyId ?? null,
|
||||
worktreeId: expectedWorktreeId
|
||||
}))
|
||||
this.terminalPaneRecoveryByIdentity.set(recoveryKey, recovery)
|
||||
const clearRecovery = (): void => {
|
||||
if (this.terminalPaneRecoveryByIdentity.get(recoveryKey) === recovery) {
|
||||
this.terminalPaneRecoveryByIdentity.delete(recoveryKey)
|
||||
}
|
||||
}
|
||||
void recovery.then(clearRecovery, clearRecovery)
|
||||
return recovery
|
||||
}
|
||||
|
||||
async showTerminal(handle: string): Promise<RuntimeTerminalShow> {
|
||||
@@ -30338,20 +30370,13 @@ export class OrcaRuntimeService {
|
||||
// The sync hasPty rescue closes the spawn/list race: a just-spawned PTY can
|
||||
// register after the inventory snapshot, and federation reads one
|
||||
// connected:false as exited.
|
||||
let directLiveness: boolean | null | undefined
|
||||
try {
|
||||
directLiveness = leaf.ptyId ? this.ptyController?.hasPty?.(leaf.ptyId) : undefined
|
||||
} catch {
|
||||
directLiveness = null
|
||||
}
|
||||
const provenAbsent =
|
||||
provenLivePtyIds !== null &&
|
||||
leaf.ptyId !== null &&
|
||||
!provenLivePtyIds.has(leaf.ptyId) &&
|
||||
!leaf.ptyId.startsWith('remote:') &&
|
||||
parseAppSshPtyId(leaf.ptyId) === null &&
|
||||
directLiveness !== true &&
|
||||
directLiveness !== null
|
||||
this.ptyController?.hasPty?.(leaf.ptyId) !== true
|
||||
return {
|
||||
handle: this.issueHandle(leaf),
|
||||
ptyId: leaf.ptyId,
|
||||
@@ -37605,14 +37630,16 @@ function runtimePathsEqual(left: string, right: string): boolean {
|
||||
* Windows/WSL/SSH ids still match themselves across hosts.
|
||||
*/
|
||||
function runtimeWorktreeIdsEqual(left: string, right: string): boolean {
|
||||
// Why: derived from the key rather than re-parsed, so equality and the sleep /
|
||||
// mutation-queue keying can never drift apart into two different identity rules.
|
||||
return runtimeWorktreeIdentityKey(left) === runtimeWorktreeIdentityKey(right)
|
||||
const parsedLeft = splitWorktreeId(left)
|
||||
const parsedRight = splitWorktreeId(right)
|
||||
return parsedLeft && parsedRight
|
||||
? parsedLeft.repoId === parsedRight.repoId &&
|
||||
runtimePathsEqual(parsedLeft.worktreePath, parsedRight.worktreePath)
|
||||
: left === right
|
||||
}
|
||||
|
||||
function runtimeWorktreeIdentityKey(worktreeId: string): string {
|
||||
// Same suffix rule: this keys PTY refresh, sleep, and mutation-queue state per session.
|
||||
// NUL cannot occur in a repoId or a path, so the joined key is unambiguous.
|
||||
const parsed = splitWorktreeId(worktreeId)
|
||||
return parsed
|
||||
? `${parsed.repoId}\0${normalizeRuntimePathForComparison(parsed.worktreePath)}`
|
||||
|
||||
@@ -39,7 +39,7 @@ import type {
|
||||
} from './types'
|
||||
import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display'
|
||||
import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-contract'
|
||||
import { isEquivalentPaneKey } from '../../../shared/stable-pane-id'
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import { OrchestrationError } from './orchestration-error'
|
||||
import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew'
|
||||
import {
|
||||
@@ -66,6 +66,16 @@ import {
|
||||
migrateMutationReceiptCapacity
|
||||
} from './mutation-receipt-capacity'
|
||||
|
||||
// Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys.
|
||||
function isEquivalentPaneKey(a: string, b: string): boolean {
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
function parseWorkerTerminalPriorOwnerIds(value: string): string[] | null {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(value)
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
import type { OrchestrationDb } from './db'
|
||||
import type { MessageRow, WorkerReportOutcome } from './types'
|
||||
import { isEquivalentPaneKey } from '../../../shared/stable-pane-id'
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
|
||||
// Why: the tab half can change on pane break-out, while opaque legacy keys
|
||||
// have no safe equivalence beyond exact equality.
|
||||
function isSamePane(assigneePaneKey: string, senderPaneKey: string): boolean {
|
||||
if (assigneePaneKey === senderPaneKey) {
|
||||
return true
|
||||
}
|
||||
const assigneeLeaf = parsePaneKey(assigneePaneKey)?.leafId
|
||||
const senderLeaf = parsePaneKey(senderPaneKey)?.leafId
|
||||
return Boolean(assigneeLeaf && senderLeaf && assigneeLeaf === senderLeaf)
|
||||
}
|
||||
|
||||
function hasLifecycleAuthority(
|
||||
dispatch: { assignee_handle: string | null; assignee_pane_key: string | null },
|
||||
@@ -8,7 +19,7 @@ function hasLifecycleAuthority(
|
||||
): boolean {
|
||||
if (dispatch.assignee_pane_key) {
|
||||
return Boolean(
|
||||
msg.sender_pane_key && isEquivalentPaneKey(dispatch.assignee_pane_key, msg.sender_pane_key)
|
||||
msg.sender_pane_key && isSamePane(dispatch.assignee_pane_key, msg.sender_pane_key)
|
||||
)
|
||||
}
|
||||
// Why: rows created before pane identity existed can only use the exact
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { isEquivalentPaneKey } from '../../../shared/stable-pane-id'
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import { OrchestrationError } from '../orchestration/orchestration-error'
|
||||
|
||||
// Why: legacy rows carry nullable pane keys, and a missing key is never proof of
|
||||
// identity — so absence is refused here rather than inside the shared comparison.
|
||||
export function equivalentLegacyPaneKey(
|
||||
a: string | null | undefined,
|
||||
b: string | null | undefined
|
||||
@@ -10,7 +8,12 @@ export function equivalentLegacyPaneKey(
|
||||
if (!a || !b) {
|
||||
return false
|
||||
}
|
||||
return isEquivalentPaneKey(a, b)
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
export function legacyReadOnlyError(): OrchestrationError {
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
/**
|
||||
* Characterisation oracle for the 30-second SSH pane recovery grant in
|
||||
* `OrcaRuntimeService.recoverTerminalPane`.
|
||||
*
|
||||
* The grant is the branch that, for a DISCONNECTED pane backed by a recently
|
||||
* expired SSH lease, calls `createTerminal` to spawn a replacement shell. This
|
||||
* file does not argue that the grant is wrong; it RECORDS that the branch is
|
||||
* already dead at the id shapes production mints, so that deleting it is
|
||||
* provably inert rather than assumed to be. Test 1 is written to survive that
|
||||
* deletion unchanged: `terminal_not_recoverable` is the answer both before and
|
||||
* after, and the anti-vacuity assertions pin that control genuinely reaches the
|
||||
* grant gate instead of bailing out earlier.
|
||||
*
|
||||
* Why the branch is dead: the gate compares `pty.ptyId` (runtime, APP form —
|
||||
* `SshPtyProvider` hands ids back through `toAppSshPtyId`, and that is the id
|
||||
* `ipc/pty.ts` passes to `runtime.registerPty`) against `lease.ptyId` (durable,
|
||||
* RELAY-native — `persistence.ts` normalizes every stored lease through
|
||||
* `toRelaySshPtyId`). The comparison at `getRecentExpiredSshLease` is a raw
|
||||
* `===` with no namespace normalization, so the two strings can never be equal.
|
||||
*
|
||||
* Honest limit on the claim. For an SSH pane the refusal is unreachable BY
|
||||
* CONSTRUCTION: an app-form id always carries the `ssh:<conn>@@` prefix that
|
||||
* the relay-native lease id, by definition, does not. For a LOCAL pane the
|
||||
* argument is weaker — a local pty id is not namespaced, so the gate is
|
||||
* unreachable only up to a random-UUID string collision with a stored lease id
|
||||
* that also matches the worktree, tab and leaf. That is not a construction
|
||||
* proof, just an astronomically unlikely one. Do not overclaim it.
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id'
|
||||
import { OrcaRuntimeService } from './orca-runtime'
|
||||
|
||||
const CONNECTION_ID = 'ssh-target-1'
|
||||
const RELAY_PTY_ID = 'pty-7'
|
||||
// Both id forms come from the production helpers so this oracle tracks the real
|
||||
// namespace split rather than restating it as two hand-typed literals.
|
||||
const APP_PTY_ID = toAppSshPtyId(CONNECTION_ID, RELAY_PTY_ID)
|
||||
const STORED_LEASE_PTY_ID = toRelaySshPtyId(CONNECTION_ID, APP_PTY_ID)
|
||||
|
||||
const REPO_ID = 'repo-1'
|
||||
const WORKTREE_PATH = '/tmp/orca-grant-reachability'
|
||||
const WORKTREE_ID = `${REPO_ID}::${WORKTREE_PATH}`
|
||||
const TAB_ID = 'tab-grant'
|
||||
// Persistence drops any leafId that is not a terminal leaf UUID, and the gate
|
||||
// compares it, so the lease only qualifies with a real one.
|
||||
const LEAF_ID = '11111111-1111-4111-8111-111111111111'
|
||||
const PANE_KEY = `${TAB_ID}:${LEAF_ID}`
|
||||
|
||||
function createStore() {
|
||||
const now = Date.now()
|
||||
return {
|
||||
getRepo: (id: string) => createStore.repos.find((repo) => repo.id === id),
|
||||
getRepos: () => createStore.repos,
|
||||
getAllWorktreeMeta: () => ({
|
||||
[WORKTREE_ID]: {
|
||||
displayName: 'grant',
|
||||
comment: '',
|
||||
linkedIssue: null,
|
||||
linkedPR: null,
|
||||
linkedLinearIssue: null,
|
||||
linkedGitLabMR: null,
|
||||
linkedGitLabIssue: null,
|
||||
isArchived: false,
|
||||
isUnread: false,
|
||||
isPinned: false,
|
||||
sortOrder: 0,
|
||||
lastActivityAt: 0
|
||||
}
|
||||
}),
|
||||
getWorktreeMeta: () => undefined,
|
||||
getSettings: () => ({
|
||||
workspaceDir: '/tmp/workspaces',
|
||||
nestWorkspaces: false,
|
||||
refreshLocalBaseRefOnWorktreeCreate: false,
|
||||
branchPrefix: 'none',
|
||||
branchPrefixCustom: ''
|
||||
}),
|
||||
getProjects: () => [],
|
||||
getSparsePresets: () => [],
|
||||
// Seeded to qualify on EVERY predicate of getRecentExpiredSshLease except
|
||||
// the ptyId comparison: expired, same worktree/tab/leaf, inside the grace.
|
||||
getSshRemotePtyLeases: () => [
|
||||
{
|
||||
targetId: CONNECTION_ID,
|
||||
ptyId: STORED_LEASE_PTY_ID,
|
||||
worktreeId: WORKTREE_ID,
|
||||
tabId: TAB_ID,
|
||||
leafId: LEAF_ID,
|
||||
state: 'expired' as const,
|
||||
createdAt: now,
|
||||
updatedAt: now
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
createStore.repos = [
|
||||
{ id: REPO_ID, path: WORKTREE_PATH, displayName: 'repo', badgeColor: 'blue', addedAt: 1 }
|
||||
]
|
||||
|
||||
/** Register one SSH-owned pty under `runtimePtyId`, then kill its shell. */
|
||||
function createDisconnectedPane(runtimePtyId: string) {
|
||||
const runtime = new OrcaRuntimeService(createStore() as never)
|
||||
runtime.registerPty(runtimePtyId, WORKTREE_ID, CONNECTION_ID, {
|
||||
tabId: TAB_ID,
|
||||
leafId: LEAF_ID
|
||||
})
|
||||
const handle = runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).handle
|
||||
runtime.onPtyExit(runtimePtyId, -1)
|
||||
return { runtime, handle }
|
||||
}
|
||||
|
||||
describe('SSH pane recovery grant reachability', () => {
|
||||
it('refuses a pane whose SSH shell died, at the id shapes production actually mints', async () => {
|
||||
// The split itself, named: the durable lease id and the runtime pty id for
|
||||
// the SAME shell are different strings.
|
||||
expect(STORED_LEASE_PTY_ID).not.toBe(APP_PTY_ID)
|
||||
|
||||
const { runtime, handle } = createDisconnectedPane(APP_PTY_ID)
|
||||
const createTerminal = vi.spyOn(runtime, 'createTerminal')
|
||||
|
||||
// Anti-vacuity: control must actually arrive at the grant gate. A truthy
|
||||
// handle rules out an early `terminal_not_found`; connected === false rules
|
||||
// out the still-connected branch that also throws not_recoverable.
|
||||
expect(handle).toBeTruthy()
|
||||
expect(runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).connected).toBe(false)
|
||||
|
||||
await expect(runtime.recoverTerminalPane(PANE_KEY, WORKTREE_ID, handle)).rejects.toThrow(
|
||||
'terminal_not_recoverable'
|
||||
)
|
||||
expect(createTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// INVERTED BY THE DELETION, and that inversion is the point.
|
||||
//
|
||||
// Before the grant was removed, this case RESOLVED with a replacement: the same seeded lease,
|
||||
// with the runtime pty registered under the bare relay id instead of the app-form one, opened
|
||||
// the gate. That is what made the case above attributable to the namespace and to nothing else —
|
||||
// the lease qualified on state, worktree, tab, leaf and grace window, and only the id comparison
|
||||
// stood between it and a spawn.
|
||||
//
|
||||
// The shape was never reachable in production (a lease implies a truthy connectionId, which
|
||||
// implies an app-form id), so flipping it to a refusal changes no real behaviour — which is
|
||||
// exactly what "the deletion is inert" means. Kept rather than dropped so the counterfactual
|
||||
// stays on record: no id shape authorizes a spawn now, not merely the one production mints.
|
||||
it('refuses even the id shape that used to open the grant', async () => {
|
||||
const { runtime, handle } = createDisconnectedPane(RELAY_PTY_ID)
|
||||
const createTerminal = vi.spyOn(runtime, 'createTerminal')
|
||||
|
||||
expect(handle).toBeTruthy()
|
||||
expect(runtime.resolveTerminalPane(PANE_KEY, WORKTREE_ID).connected).toBe(false)
|
||||
|
||||
await expect(runtime.recoverTerminalPane(PANE_KEY, WORKTREE_ID, handle)).rejects.toThrow(
|
||||
'terminal_not_recoverable'
|
||||
)
|
||||
expect(createTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -148,23 +148,6 @@ describe('listTerminals liveness truth for restored leaves', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps a leaf connected when the snapshot missed it but provider liveness is unknown', async () => {
|
||||
const runtime = makeRuntimeWithLeaf({
|
||||
leafPtyId: 'pty-provider-disconnected',
|
||||
controllerSessions: [],
|
||||
hasPty: () => null
|
||||
})
|
||||
|
||||
const { terminals } = await runtime.listTerminals(`id:${WORKTREE_ID}`)
|
||||
|
||||
expect(terminals).toHaveLength(1)
|
||||
expect(terminals[0]).toMatchObject({
|
||||
ptyId: 'pty-provider-disconnected',
|
||||
connected: true,
|
||||
writable: true
|
||||
})
|
||||
})
|
||||
|
||||
it('does not demote remote-runtime-scoped leaves the local inventory never covers', async () => {
|
||||
const runtime = makeRuntimeWithLeaf({
|
||||
leafPtyId: 'remote:env-1@@term_abc',
|
||||
|
||||
@@ -1,365 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { FLOATING_TERMINAL_WORKTREE_ID, getDefaultWorkspaceSession } from '../../shared/constants'
|
||||
import { makePaneKey } from '../../shared/stable-pane-id'
|
||||
import type { WorkspaceSessionState } from '../../shared/types'
|
||||
import { FOLDER_WORKSPACE_INSTANCE_SEPARATOR } from '../../shared/worktree-id'
|
||||
import { OrcaRuntimeService } from './orca-runtime'
|
||||
|
||||
/**
|
||||
* Journey: git-worktree, folder-workspace and floating namespaces must all resolve
|
||||
* terminal identity from one stable host-local workspace id. Neither the filesystem
|
||||
* path, the client's repository id, nor the caller's target id may stand in for it.
|
||||
* Drive-letter and UNC ids normalize host-independently and are covered here; the
|
||||
* Windows PTY lifecycle behind them is not.
|
||||
*/
|
||||
|
||||
const GIT_REPO_ID = 'repo-git'
|
||||
const GIT_ROOT_PATH = '/tmp/ns/checkout'
|
||||
const GIT_ROOT_ID = `${GIT_REPO_ID}::${GIT_ROOT_PATH}`
|
||||
const GIT_CHILD_PATH = `${GIT_ROOT_PATH}/.claude/worktrees/child`
|
||||
const GIT_CHILD_ID = `${GIT_REPO_ID}::${GIT_CHILD_PATH}`
|
||||
|
||||
const FOLDER_REPO_ID = 'repo-folder'
|
||||
const FOLDER_PATH = '/tmp/ns/folder'
|
||||
const FOLDER_ROOT_ID = `${FOLDER_REPO_ID}::${FOLDER_PATH}`
|
||||
const FOLDER_A = `${FOLDER_ROOT_ID}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa`
|
||||
const FOLDER_B = `${FOLDER_ROOT_ID}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb`
|
||||
|
||||
// Same directory, re-added as a second project: only the repo id differs.
|
||||
const ALIAS_REPO_ID = 'repo-folder-alias'
|
||||
const ALIAS_ID = `${ALIAS_REPO_ID}::${FOLDER_PATH}`
|
||||
|
||||
const PTY_GIT_ROOT = `${GIT_ROOT_ID}@@pty-git-root`
|
||||
const PTY_GIT_CHILD = `${GIT_CHILD_ID}@@pty-git-child`
|
||||
const PTY_A = `${FOLDER_A}@@pty-a`
|
||||
const PTY_B = `${FOLDER_B}@@pty-b`
|
||||
const PTY_FOLDER_ROOT = `${FOLDER_ROOT_ID}@@pty-folder-root`
|
||||
const PTY_ALIAS = `${ALIAS_ID}@@pty-alias`
|
||||
const PTY_FLOATING = 'floating-pty'
|
||||
|
||||
const LEAF_A = 'a1111111-1111-4111-8111-111111111111'
|
||||
const LEAF_B = 'b2222222-2222-4222-8222-222222222222'
|
||||
|
||||
const REPOS = [
|
||||
{ id: GIT_REPO_ID, path: GIT_ROOT_PATH, displayName: 'checkout', badgeColor: 'blue', addedAt: 1 },
|
||||
{
|
||||
id: FOLDER_REPO_ID,
|
||||
path: FOLDER_PATH,
|
||||
displayName: 'folder',
|
||||
badgeColor: 'green',
|
||||
addedAt: 2,
|
||||
kind: 'folder'
|
||||
},
|
||||
{
|
||||
id: ALIAS_REPO_ID,
|
||||
path: FOLDER_PATH,
|
||||
displayName: 'folder-alias',
|
||||
badgeColor: 'red',
|
||||
addedAt: 3,
|
||||
kind: 'folder'
|
||||
}
|
||||
] as const
|
||||
|
||||
const ALL_WORKSPACE_IDS = [GIT_ROOT_ID, GIT_CHILD_ID, FOLDER_ROOT_ID, FOLDER_A, FOLDER_B, ALIAS_ID]
|
||||
|
||||
type ControllerSession = {
|
||||
id: string
|
||||
worktreeId?: string
|
||||
cwd?: string
|
||||
title?: string
|
||||
incarnationId?: string
|
||||
terminalHandle?: string
|
||||
}
|
||||
|
||||
type RuntimeInternals = {
|
||||
buildResolvedWorktreeFromId: (worktreeId: string) => unknown
|
||||
refreshPtyWorktreeRecordsWithControllerInventory: (
|
||||
resolvedWorktrees: unknown[],
|
||||
targetWorktreeId?: string | null
|
||||
) => Promise<{ livePtyIds: Set<string> } | null>
|
||||
getLivePtyIdsForWorktree: (worktreeId: string) => Set<string>
|
||||
hasExactPersistedTerminalSurfaceIdentity: (expected: {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
incarnationId: string
|
||||
}) => boolean
|
||||
ptysById: Map<string, { worktreeId: string; connected: boolean }>
|
||||
recordPtyWorktree: (ptyId: string, worktreeId: string, state?: Record<string, unknown>) => unknown
|
||||
}
|
||||
|
||||
function createRuntimeInternals(options: {
|
||||
session?: WorkspaceSessionState
|
||||
sessions?: ControllerSession[]
|
||||
}): RuntimeInternals {
|
||||
const meta = Object.fromEntries(ALL_WORKSPACE_IDS.map((id) => [id, { hostId: 'local' }]))
|
||||
const store = {
|
||||
getRepos: () => REPOS,
|
||||
getRepo: (id: string) => REPOS.find((repo) => repo.id === id),
|
||||
getAllWorktreeMeta: () => meta,
|
||||
getWorktreeMeta: (worktreeId: string) => meta[worktreeId],
|
||||
setWorktreeMeta: (worktreeId: string, patch: Record<string, unknown>) => {
|
||||
meta[worktreeId] = { ...meta[worktreeId], ...patch }
|
||||
return meta[worktreeId]
|
||||
},
|
||||
getWorkspaceSession: () => options.session ?? getDefaultWorkspaceSession(),
|
||||
setWorkspaceSession: () => {},
|
||||
flushOrThrow: () => {}
|
||||
} as never
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.setPtyController({
|
||||
write: () => true,
|
||||
kill: () => true,
|
||||
stopAndWait: async () => true,
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => options.sessions ?? []
|
||||
} as never)
|
||||
return runtime as unknown as RuntimeInternals
|
||||
}
|
||||
|
||||
function resolveWorktrees(internals: RuntimeInternals, ids: readonly string[]): unknown[] {
|
||||
return ids.map((id) => internals.buildResolvedWorktreeFromId(id))
|
||||
}
|
||||
|
||||
/** Persisted layout binding each folder sibling's own PTY to its own tab and leaf. */
|
||||
function persistedFolderSiblingSession(): WorkspaceSessionState {
|
||||
return {
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: {
|
||||
[FOLDER_A]: [{ id: 'tab-a', worktreeId: FOLDER_A, title: 'A' }],
|
||||
[FOLDER_B]: [{ id: 'tab-b', worktreeId: FOLDER_B, title: 'B' }]
|
||||
} as never,
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-a': { root: null, activeLeafId: LEAF_A, ptyIdsByLeafId: { [LEAF_A]: PTY_A } },
|
||||
'tab-b': { root: null, activeLeafId: LEAF_B, ptyIdsByLeafId: { [LEAF_B]: PTY_B } }
|
||||
} as never,
|
||||
terminalPtyIncarnationsByPaneKey: {
|
||||
[makePaneKey('tab-a', LEAF_A)]: 'inc-a',
|
||||
[makePaneKey('tab-b', LEAF_B)]: 'inc-b'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('workspace namespaces keep terminal identity host-local and per-workspace', () => {
|
||||
describe('git worktree namespace', () => {
|
||||
it('binds a nested worktree PTY to the nested worktree, not the targeted parent', async () => {
|
||||
// cwd alone cannot separate a parent checkout from a worktree nested inside it.
|
||||
const internals = createRuntimeInternals({
|
||||
sessions: [{ id: 'unattributed-pty', cwd: `${GIT_CHILD_PATH}/src`, title: 'child work' }]
|
||||
})
|
||||
const resolved = resolveWorktrees(internals, [GIT_ROOT_ID, GIT_CHILD_ID])
|
||||
|
||||
const scopedToParent = await internals.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolved,
|
||||
GIT_ROOT_ID
|
||||
)
|
||||
const scopedToChild = await internals.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolved,
|
||||
GIT_CHILD_ID
|
||||
)
|
||||
|
||||
expect([...(scopedToParent?.livePtyIds ?? [])]).toEqual([])
|
||||
expect([...(scopedToChild?.livePtyIds ?? [])]).toEqual(['unattributed-pty'])
|
||||
expect(internals.ptysById.get('unattributed-pty')?.worktreeId).toBe(GIT_CHILD_ID)
|
||||
})
|
||||
|
||||
it('keeps a checkout and the worktree nested inside it in separate namespaces', async () => {
|
||||
const internals = createRuntimeInternals({
|
||||
sessions: [
|
||||
{ id: PTY_GIT_ROOT, worktreeId: GIT_ROOT_ID, cwd: GIT_ROOT_PATH, title: 'git' },
|
||||
{ id: PTY_GIT_CHILD, worktreeId: GIT_CHILD_ID, cwd: GIT_CHILD_PATH, title: 'child' }
|
||||
]
|
||||
})
|
||||
const resolved = resolveWorktrees(internals, [GIT_ROOT_ID, GIT_CHILD_ID])
|
||||
|
||||
await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved)
|
||||
|
||||
expect(internals.ptysById.get(PTY_GIT_ROOT)?.worktreeId).toBe(GIT_ROOT_ID)
|
||||
expect(internals.ptysById.get(PTY_GIT_CHILD)?.worktreeId).toBe(GIT_CHILD_ID)
|
||||
expect([...internals.getLivePtyIdsForWorktree(GIT_ROOT_ID)]).toEqual([PTY_GIT_ROOT])
|
||||
expect([...internals.getLivePtyIdsForWorktree(GIT_CHILD_ID)]).toEqual([PTY_GIT_CHILD])
|
||||
})
|
||||
})
|
||||
|
||||
describe('folder workspace namespace', () => {
|
||||
it('does not let the caller target stand in for an attributed PTY owner', async () => {
|
||||
const internals = createRuntimeInternals({
|
||||
sessions: [{ id: PTY_B, worktreeId: FOLDER_B, cwd: FOLDER_PATH, title: 'b' }]
|
||||
})
|
||||
const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B])
|
||||
|
||||
await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved)
|
||||
const scopedToA = await internals.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolved,
|
||||
FOLDER_A
|
||||
)
|
||||
|
||||
expect([...(scopedToA?.livePtyIds ?? [])]).toEqual([])
|
||||
expect(internals.ptysById.get(PTY_B)?.worktreeId).toBe(FOLDER_B)
|
||||
})
|
||||
|
||||
it('keeps each sibling terminal bound to its own workspace across a reconnect', async () => {
|
||||
const sessions: ControllerSession[] = [
|
||||
{
|
||||
id: PTY_A,
|
||||
worktreeId: FOLDER_A,
|
||||
cwd: FOLDER_PATH,
|
||||
title: 'a',
|
||||
incarnationId: 'inc-a',
|
||||
terminalHandle: 'term_a'
|
||||
},
|
||||
{
|
||||
id: PTY_B,
|
||||
worktreeId: FOLDER_B,
|
||||
cwd: FOLDER_PATH,
|
||||
title: 'b',
|
||||
incarnationId: 'inc-b',
|
||||
terminalHandle: 'term_b'
|
||||
}
|
||||
]
|
||||
const internals = createRuntimeInternals({
|
||||
session: persistedFolderSiblingSession(),
|
||||
sessions
|
||||
})
|
||||
const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B])
|
||||
|
||||
// Reconnect resync: each workspace re-lists the same controller inventory.
|
||||
const afterA = await internals.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolved,
|
||||
FOLDER_A
|
||||
)
|
||||
const afterB = await internals.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolved,
|
||||
FOLDER_B
|
||||
)
|
||||
|
||||
expect([...(afterA?.livePtyIds ?? [])]).toEqual([PTY_A])
|
||||
expect([...(afterB?.livePtyIds ?? [])]).toEqual([PTY_B])
|
||||
expect(internals.ptysById.get(PTY_A)?.worktreeId).toBe(FOLDER_A)
|
||||
expect(internals.ptysById.get(PTY_B)?.worktreeId).toBe(FOLDER_B)
|
||||
expect(
|
||||
internals.hasExactPersistedTerminalSurfaceIdentity({
|
||||
worktreeId: FOLDER_B,
|
||||
tabId: 'tab-b',
|
||||
leafId: LEAF_B,
|
||||
ptyId: PTY_B,
|
||||
incarnationId: 'inc-b'
|
||||
})
|
||||
).toBe(true)
|
||||
// A sibling must never satisfy the other's persisted surface identity.
|
||||
expect(
|
||||
internals.hasExactPersistedTerminalSurfaceIdentity({
|
||||
worktreeId: FOLDER_A,
|
||||
tabId: 'tab-b',
|
||||
leafId: LEAF_B,
|
||||
ptyId: PTY_B,
|
||||
incarnationId: 'inc-b'
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('does not treat one directory as one namespace across two projects', async () => {
|
||||
// Same directory added twice: the ids differ only by the client's repo id.
|
||||
const internals = createRuntimeInternals({
|
||||
sessions: [
|
||||
{ id: PTY_A, worktreeId: FOLDER_A, cwd: FOLDER_PATH, title: 'a' },
|
||||
{ id: PTY_FOLDER_ROOT, worktreeId: FOLDER_ROOT_ID, cwd: FOLDER_PATH, title: 'root' },
|
||||
{ id: PTY_ALIAS, worktreeId: ALIAS_ID, cwd: FOLDER_PATH, title: 'alias' }
|
||||
]
|
||||
})
|
||||
const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B, FOLDER_ROOT_ID, ALIAS_ID])
|
||||
|
||||
await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved)
|
||||
|
||||
expect(internals.ptysById.get(PTY_A)?.worktreeId).toBe(FOLDER_A)
|
||||
expect(internals.ptysById.get(PTY_FOLDER_ROOT)?.worktreeId).toBe(FOLDER_ROOT_ID)
|
||||
expect(internals.ptysById.get(PTY_ALIAS)?.worktreeId).toBe(ALIAS_ID)
|
||||
expect([...internals.getLivePtyIdsForWorktree(ALIAS_ID)]).toEqual([PTY_ALIAS])
|
||||
expect([...internals.getLivePtyIdsForWorktree(FOLDER_ROOT_ID)]).toEqual([PTY_FOLDER_ROOT])
|
||||
expect([...internals.getLivePtyIdsForWorktree(FOLDER_A)]).toEqual([PTY_A])
|
||||
expect([...internals.getLivePtyIdsForWorktree(FOLDER_B)]).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('floating workspace namespace', () => {
|
||||
it('never lets a repo workspace claim a floating terminal or vice versa', async () => {
|
||||
const internals = createRuntimeInternals({
|
||||
sessions: [
|
||||
{
|
||||
id: PTY_FLOATING,
|
||||
worktreeId: FLOATING_TERMINAL_WORKTREE_ID,
|
||||
cwd: FOLDER_PATH,
|
||||
title: 'floating'
|
||||
},
|
||||
{ id: PTY_A, worktreeId: FOLDER_A, cwd: FOLDER_PATH, title: 'a' }
|
||||
]
|
||||
})
|
||||
const resolved = resolveWorktrees(internals, [FOLDER_A, FOLDER_B])
|
||||
|
||||
await internals.refreshPtyWorktreeRecordsWithControllerInventory(resolved)
|
||||
const scopedToA = await internals.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolved,
|
||||
FOLDER_A
|
||||
)
|
||||
|
||||
// The floating id carries no repo/path pair, so it can only equal itself.
|
||||
expect([...(scopedToA?.livePtyIds ?? [])]).toEqual([PTY_A])
|
||||
expect(internals.ptysById.get(PTY_FLOATING)?.worktreeId).toBe(FLOATING_TERMINAL_WORKTREE_ID)
|
||||
expect([...internals.getLivePtyIdsForWorktree(FLOATING_TERMINAL_WORKTREE_ID)]).toEqual([
|
||||
PTY_FLOATING
|
||||
])
|
||||
expect([...internals.getLivePtyIdsForWorktree(FOLDER_A)]).toEqual([PTY_A])
|
||||
})
|
||||
|
||||
it('does not resolve a floating terminal onto a workspace that shares its cwd', () => {
|
||||
const internals = createRuntimeInternals({})
|
||||
internals.recordPtyWorktree(PTY_FLOATING, FLOATING_TERMINAL_WORKTREE_ID, { connected: true })
|
||||
internals.recordPtyWorktree(PTY_A, FOLDER_A, { connected: true })
|
||||
|
||||
expect([...internals.getLivePtyIdsForWorktree(FOLDER_ROOT_ID)]).toEqual([])
|
||||
expect([...internals.getLivePtyIdsForWorktree(FLOATING_TERMINAL_WORKTREE_ID)]).toEqual([
|
||||
PTY_FLOATING
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
// Only the id-normalization half of the Windows namespaces is host-independent.
|
||||
// Spawning and reconnecting a real drive/UNC PTY still needs a Windows host.
|
||||
describe('drive-letter and UNC namespaces (id normalization only)', () => {
|
||||
it('folds drive spelling and WSL UNC aliases but keeps distinct roots apart', () => {
|
||||
const internals = createRuntimeInternals({})
|
||||
internals.recordPtyWorktree('pty-c', 'repo-win::C:\\ws\\proj', { connected: true })
|
||||
internals.recordPtyWorktree('pty-d', 'repo-win::D:\\ws\\proj', { connected: true })
|
||||
internals.recordPtyWorktree('pty-share', 'repo-win::\\\\server\\share\\proj', {
|
||||
connected: true
|
||||
})
|
||||
internals.recordPtyWorktree('pty-other-share', 'repo-win::\\\\server\\other\\proj', {
|
||||
connected: true
|
||||
})
|
||||
internals.recordPtyWorktree('pty-wsl', 'repo-win::\\\\wsl$\\Ubuntu\\home\\me', {
|
||||
connected: true
|
||||
})
|
||||
|
||||
expect([...internals.getLivePtyIdsForWorktree('repo-win::c:/ws/proj')]).toEqual(['pty-c'])
|
||||
expect([...internals.getLivePtyIdsForWorktree('repo-win::D:/ws/proj')]).toEqual(['pty-d'])
|
||||
expect([...internals.getLivePtyIdsForWorktree('repo-win::\\\\server\\share\\proj')]).toEqual([
|
||||
'pty-share'
|
||||
])
|
||||
expect([
|
||||
...internals.getLivePtyIdsForWorktree('repo-win::\\\\wsl.localhost\\ubuntu\\home\\me')
|
||||
]).toEqual(['pty-wsl'])
|
||||
})
|
||||
|
||||
it('keeps a folder workspace instance on a drive path distinct from its root', () => {
|
||||
const driveRoot = 'repo-win::C:\\ws\\proj'
|
||||
const driveInstance = `${driveRoot}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}dddddddd-dddd-4ddd-8ddd-dddddddddddd`
|
||||
const internals = createRuntimeInternals({})
|
||||
internals.recordPtyWorktree('pty-drive-root', driveRoot, { connected: true })
|
||||
internals.recordPtyWorktree('pty-drive-instance', driveInstance, { connected: true })
|
||||
|
||||
expect([...internals.getLivePtyIdsForWorktree(driveRoot)]).toEqual(['pty-drive-root'])
|
||||
expect([...internals.getLivePtyIdsForWorktree(driveInstance)]).toEqual(['pty-drive-instance'])
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -161,30 +161,6 @@ export function advanceTerminalTopologyRevision(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The tab whose live layout holds this leaf. Only the leaf half of a pane key is stable — breaking
|
||||
* a pane out into its own tab moves the leaf and leaves any stored tabId naming the tab it left.
|
||||
*/
|
||||
export function findTerminalTabIdForLeaf(
|
||||
session: WorkspaceSessionState | undefined,
|
||||
leafId: string
|
||||
): string | undefined {
|
||||
// Only a still-existing tab counts: a layout entry outlives its tab, and binding a live shell to
|
||||
// a deleted one registers a pane under a ghost. Scanned across worktrees rather than under one
|
||||
// key so the answer cannot hinge on a worktreeId matching exactly (`::workspace:` suffixes).
|
||||
const liveTabIds = new Set(
|
||||
Object.values(session?.tabsByWorktree ?? {}).flatMap((tabs) => tabs.map((tab) => tab.id))
|
||||
)
|
||||
for (const [tabId, layout] of Object.entries(session?.terminalLayoutsByTabId ?? {})) {
|
||||
const leafIds = new Set<string>()
|
||||
collectLeafIds(layout.root, leafIds)
|
||||
if (liveTabIds.has(tabId) && leafIds.has(leafId)) {
|
||||
return tabId
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function hasHostAuthoritativeTerminalMembership(
|
||||
session: WorkspaceSessionState | undefined,
|
||||
worktreeId: string
|
||||
|
||||
@@ -1,399 +0,0 @@
|
||||
/**
|
||||
* STA-3077 step P: an SSH pane's durable binding must live in ONE partition.
|
||||
*
|
||||
* Today it lives in two. Main's spawn writes `ssh:<target>`
|
||||
* (ipc/pty.ts persistPtyBinding(binding, toSshExecutionHostId(connectionId))),
|
||||
* the relay's reattach write passes no hostId and lands in `local`
|
||||
* (ssh-relay-session.ts restoreReattachedPtyRuntime), and the renderer keeps SSH
|
||||
* worktrees in `local` on purpose (buildHostIdByWorktreeId). Supersession then
|
||||
* reads a binding no live writer maintains, so it compares the arriving lease
|
||||
* against a stale pty id, bails, and leaves the predecessor live. That is the
|
||||
* reported 2 -> 19 -> 20 mechanism.
|
||||
*
|
||||
* These oracles assert cardinality and identity — one pane, one live claim, and
|
||||
* the surviving claim is the shell the pane is bound to — never which function
|
||||
* ran, so they stay valid under any single-accessor implementation.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { getDefaultPersistedState, getDefaultWorkspaceSession } from '../shared/constants'
|
||||
import { LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId } from '../shared/execution-host'
|
||||
import { toAppSshPtyId, toRelaySshPtyId } from '../shared/ssh-pty-id'
|
||||
|
||||
const testState = { dir: '' }
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: { getPath: () => testState.dir },
|
||||
safeStorage: { isEncryptionAvailable: () => false }
|
||||
}))
|
||||
vi.mock('./telemetry/client', () => ({ track: vi.fn() }))
|
||||
vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() }))
|
||||
|
||||
const TARGET = 'ssh-target-1'
|
||||
const SSH_PARTITION = toSshExecutionHostId(TARGET)
|
||||
const WORKTREE = 'repo-1:wt-1'
|
||||
const TAB = 'tab-1'
|
||||
/** Must be a real layout leaf UUID — the store drops any other spelling. */
|
||||
const LEAF = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90'
|
||||
|
||||
const appPtyId = (relayPtyId: string): string => toAppSshPtyId(TARGET, relayPtyId)
|
||||
|
||||
async function createStore(state: Record<string, unknown> = {}) {
|
||||
mkdirSync(testState.dir, { recursive: true })
|
||||
writeFileSync(
|
||||
join(testState.dir, 'orca-data.json'),
|
||||
JSON.stringify({ ...getDefaultPersistedState(testState.dir), ...state }),
|
||||
'utf-8'
|
||||
)
|
||||
vi.resetModules()
|
||||
const { Store, initDataPath } = await import('./persistence')
|
||||
initDataPath()
|
||||
return new Store()
|
||||
}
|
||||
|
||||
type TestStore = Awaited<ReturnType<typeof createStore>>
|
||||
|
||||
/** One SSH pane, bound to `relayPtyId`, as a partition stores it. */
|
||||
function paneSession(relayPtyId: string) {
|
||||
const ptyId = appPtyId(relayPtyId)
|
||||
return {
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: {
|
||||
[WORKTREE]: [
|
||||
{
|
||||
id: TAB,
|
||||
ptyId,
|
||||
worktreeId: WORKTREE,
|
||||
title: 'Terminal 1',
|
||||
defaultTitle: 'Terminal 1',
|
||||
customTitle: null,
|
||||
color: null,
|
||||
sortOrder: 0,
|
||||
createdAt: 1
|
||||
}
|
||||
]
|
||||
},
|
||||
terminalLayoutsByTabId: {
|
||||
[TAB]: {
|
||||
root: { type: 'leaf' as const, leafId: LEAF },
|
||||
activeLeafId: LEAF,
|
||||
expandedLeafId: null,
|
||||
ptyIdsByLeafId: { [LEAF]: ptyId }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** session:set for an SSH worktree — buildHostIdByWorktreeId sends it to `local`. */
|
||||
function rendererPublishesPane(store: TestStore, relayPtyId: string): void {
|
||||
store.setWorkspaceSession(paneSession(relayPtyId) as never, LOCAL_EXECUTION_HOST_ID)
|
||||
}
|
||||
|
||||
/** A mid-session write into `ssh:<target>` — orphan adoption still targets that partition, so the
|
||||
* one-time load fold cannot be the only thing keeping the two homes from reappearing. */
|
||||
function somethingRewritesTheSshPartition(store: TestStore, relayPtyId: string): void {
|
||||
store.setWorkspaceSession(paneSession(relayPtyId) as never, SSH_PARTITION)
|
||||
}
|
||||
|
||||
/** ssh-relay-session.ts:2504 — the reattach bind. No hostId, refuses to create. */
|
||||
function relayReattachBindsPane(store: TestStore, relayPtyId: string): boolean | null {
|
||||
return store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: appPtyId(relayPtyId),
|
||||
incarnationId: `inc-${relayPtyId}`,
|
||||
mayCreate: false
|
||||
})
|
||||
}
|
||||
|
||||
/** ipc/pty.ts:6493 — the spawn's lease upsert, ahead of its binding write. */
|
||||
function sshSpawnUpsertsLease(store: TestStore, relayPtyId: string): void {
|
||||
store.upsertSshRemotePtyLease({
|
||||
targetId: TARGET,
|
||||
ptyId: toRelaySshPtyId(TARGET, appPtyId(relayPtyId)),
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
state: 'attached',
|
||||
lastAttachedAt: Date.now()
|
||||
})
|
||||
}
|
||||
|
||||
function liveLeaseIdsForPane(store: TestStore): string[] {
|
||||
return store
|
||||
.getSshRemotePtyLeases(TARGET)
|
||||
.filter(
|
||||
(lease) =>
|
||||
lease.tabId === TAB &&
|
||||
lease.leafId === LEAF &&
|
||||
lease.state !== 'terminated' &&
|
||||
lease.state !== 'expired'
|
||||
)
|
||||
.map((lease) => lease.ptyId)
|
||||
.sort()
|
||||
}
|
||||
|
||||
/** The argument text of every `callee(...)` call in a production source file. */
|
||||
function callArgumentsIn(source: string, callee: string): string[] {
|
||||
const calls: string[] = []
|
||||
for (let index = source.indexOf(`${callee}(`); index !== -1; ) {
|
||||
let cursor = index + callee.length
|
||||
const start = cursor + 1
|
||||
for (let depth = 0; cursor < source.length; cursor += 1) {
|
||||
if (source[cursor] === '(') {
|
||||
depth += 1
|
||||
} else if (source[cursor] === ')' && --depth === 0) {
|
||||
break
|
||||
}
|
||||
}
|
||||
calls.push(source.slice(start, cursor))
|
||||
index = source.indexOf(`${callee}(`, cursor)
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
/** On-disk state from an earlier session: both partitions name the same shell,
|
||||
* because main's spawn wrote `ssh:<target>` and the renderer published `local`.
|
||||
* They diverge as soon as the relay's reattach write updates only `local`. */
|
||||
function diskAfterEarlierSession(relayPtyId: string) {
|
||||
return {
|
||||
workspaceSession: paneSession(relayPtyId),
|
||||
workspaceSessionsByHostId: { [SSH_PARTITION]: paneSession(relayPtyId) }
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
testState.dir = mkdtempSync(join(tmpdir(), 'orca-sta3077-partition-'))
|
||||
})
|
||||
|
||||
describe('STA-3077 step P: one pane, one live claim across partitions', () => {
|
||||
// The pane is bound to pty-2 in `local` (the relay's reattach write put it
|
||||
// there); the SSH partition still names the predecessor pty-1. Supersession
|
||||
// reads ssh-first, sees pty-1 != pty-2, and bails — both shells stay claimed.
|
||||
it('supersedes the predecessor when the pane is bound in the local partition', async () => {
|
||||
const store = await createStore(diskAfterEarlierSession('pty-1'))
|
||||
sshSpawnUpsertsLease(store, 'pty-1')
|
||||
expect(relayReattachBindsPane(store, 'pty-2')).toBe(true)
|
||||
|
||||
sshSpawnUpsertsLease(store, 'pty-2')
|
||||
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-2'])
|
||||
})
|
||||
|
||||
// reattachKnownPtys calls this first, so a wrong winner here is what fans the
|
||||
// reconnect out over a dead id and grafts a pane the user never opened.
|
||||
it('keeps the lease the live pane binding names when healing duplicates', async () => {
|
||||
const store = await createStore(diskAfterEarlierSession('pty-1'))
|
||||
sshSpawnUpsertsLease(store, 'pty-1')
|
||||
sshSpawnUpsertsLease(store, 'pty-2')
|
||||
expect(relayReattachBindsPane(store, 'pty-2')).toBe(true)
|
||||
|
||||
await store.supersedeDuplicatePaneLeases(TARGET)
|
||||
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-2'])
|
||||
})
|
||||
|
||||
// Isolates the reader from the one-time load fold. Without this clause the fold masks the
|
||||
// partition preference — it deletes the divergent copy at boot, so restoring the ssh-first
|
||||
// hedge stays green and the reader guard ships unproven. Anything that writes `ssh:<target>`
|
||||
// after load (orphan adoption still does) would then revive the defect inside one session.
|
||||
it('supersedes the predecessor when the ssh partition is rewritten mid-session', async () => {
|
||||
const store = await createStore(diskAfterEarlierSession('pty-1'))
|
||||
sshSpawnUpsertsLease(store, 'pty-1')
|
||||
expect(relayReattachBindsPane(store, 'pty-2')).toBe(true)
|
||||
somethingRewritesTheSshPartition(store, 'pty-1')
|
||||
|
||||
sshSpawnUpsertsLease(store, 'pty-2')
|
||||
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-2'])
|
||||
})
|
||||
|
||||
// The reported growth: live claims must not scale with reconnect count.
|
||||
it('holds the live claim count flat across ten reconnects of one pane', async () => {
|
||||
const store = await createStore(diskAfterEarlierSession('pty-0'))
|
||||
sshSpawnUpsertsLease(store, 'pty-0')
|
||||
|
||||
for (let reconnect = 1; reconnect <= 10; reconnect += 1) {
|
||||
expect(relayReattachBindsPane(store, `pty-${reconnect}`)).toBe(true)
|
||||
sshSpawnUpsertsLease(store, `pty-${reconnect}`)
|
||||
}
|
||||
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-10'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: a decision made from one plane only mutates that plane', () => {
|
||||
// Arbitration ranks using the desktop plane's binding. The headless plane's binding never gets a
|
||||
// vote, so deleting it here would strand a shell its owner may still be using — and that owner
|
||||
// would then have no durable record to reattach by. An explicit expiry is different: the pty is
|
||||
// gone for everyone, and that path still scrubs both.
|
||||
it('leaves the headless plane binding intact when supersession retires its lease', async () => {
|
||||
const sshSession = paneSession('pty-old') as unknown as Record<string, unknown>
|
||||
const store = await createStore({
|
||||
workspaceSession: paneSession('pty-new'),
|
||||
workspaceSessionsByHostId: { [SSH_PARTITION]: sshSession }
|
||||
})
|
||||
sshSpawnUpsertsLease(store, 'pty-old')
|
||||
|
||||
// Local names pty-new, so arbitration expires pty-old without the other plane being consulted.
|
||||
sshSpawnUpsertsLease(store, 'pty-new')
|
||||
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-new'])
|
||||
expect(
|
||||
store.getWorkspaceSession(SSH_PARTITION).terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]
|
||||
).toBe(appPtyId('pty-old'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: a pane moved between tabs still supersedes its own predecessor', () => {
|
||||
// The reported cardinality growth, in its surviving form. A lease freezes its tabId at write
|
||||
// time. Break the pane out into a new tab and its next lease carries the NEW tab, so matching
|
||||
// siblings on tabId means the two leases for one pane never meet — the predecessor is never
|
||||
// superseded and the count grows on every reconnect, exactly as reported.
|
||||
function leaseInTab(store: TestStore, relayPtyId: string, tabId: string): void {
|
||||
store.upsertSshRemotePtyLease({
|
||||
targetId: TARGET,
|
||||
ptyId: toRelaySshPtyId(TARGET, appPtyId(relayPtyId)),
|
||||
worktreeId: WORKTREE,
|
||||
tabId,
|
||||
leafId: LEAF,
|
||||
state: 'attached',
|
||||
lastAttachedAt: Date.now()
|
||||
})
|
||||
}
|
||||
const liveForLeaf = (store: TestStore): string[] =>
|
||||
store
|
||||
.getSshRemotePtyLeases(TARGET)
|
||||
.filter((l) => l.leafId === LEAF && l.state !== 'terminated' && l.state !== 'expired')
|
||||
.map((l) => l.ptyId)
|
||||
.sort()
|
||||
|
||||
it('supersedes a predecessor whose lease names the tab the pane left', async () => {
|
||||
const store = await createStore()
|
||||
leaseInTab(store, 'pty-old', TAB)
|
||||
|
||||
leaseInTab(store, 'pty-new', 'tab-moved-to')
|
||||
|
||||
expect(liveForLeaf(store)).toEqual(['pty-new'])
|
||||
})
|
||||
|
||||
it('holds the count flat across ten reconnects that each land in a new tab', async () => {
|
||||
const store = await createStore()
|
||||
leaseInTab(store, 'pty-0', TAB)
|
||||
for (let n = 1; n <= 10; n += 1) {
|
||||
leaseInTab(store, `pty-${n}`, `tab-${n}`)
|
||||
}
|
||||
|
||||
expect(liveForLeaf(store)).toEqual(['pty-10'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: arbitration follows the pane, not the tab it was written in', () => {
|
||||
// A lease freezes its tabId when written; `detachTerminalPaneToTab` then moves the live pane and
|
||||
// its PTY into a new tab. Looking the binding up under the frozen tab finds nothing, so the bound
|
||||
// shell loses to recency and supersession retires the pane's OWN shell in favour of a stale one.
|
||||
it('finds the binding after the pane has been moved to another tab', async () => {
|
||||
const MOVED_TAB = 'tab-moved-to'
|
||||
const store = await createStore()
|
||||
// The renderer publishes the pane in its NEW tab; the lease still names the old one.
|
||||
store.setWorkspaceSession(
|
||||
{
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: { [WORKTREE]: [{ id: MOVED_TAB, worktreeId: WORKTREE }] },
|
||||
terminalLayoutsByTabId: {
|
||||
[MOVED_TAB]: {
|
||||
root: { type: 'leaf' as const, leafId: LEAF },
|
||||
activeLeafId: LEAF,
|
||||
expandedLeafId: null,
|
||||
ptyIdsByLeafId: { [LEAF]: appPtyId('pty-bound') }
|
||||
}
|
||||
}
|
||||
} as never,
|
||||
LOCAL_EXECUTION_HOST_ID
|
||||
)
|
||||
sshSpawnUpsertsLease(store, 'pty-bound')
|
||||
// A newer, unbound lease arrives for the same pane under the lease's frozen tab.
|
||||
sshSpawnUpsertsLease(store, 'pty-newer')
|
||||
|
||||
await store.supersedeDuplicatePaneLeases(TARGET)
|
||||
|
||||
// The bound shell must win. Keyed on the frozen tab, it is invisible and recency retires it.
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-bound'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077 step P: the desktop plane resolves from its own home', () => {
|
||||
// INVERTED. This clause used to require the two partitions to AGREE after load, which assumed
|
||||
// `ssh:<target>` was a stale spill of this plane's state. It is not — it is the headless/CLI
|
||||
// plane's own home, which that plane writes and reads deliberately (STA-3463, STA-3465). The
|
||||
// property that actually matters is narrower and stronger: what the desktop plane resolves must
|
||||
// follow `local` alone, whatever the other plane holds. Asserting agreement made a migration look
|
||||
// necessary that in fact erased another plane's live state.
|
||||
it('resolves to the local binding regardless of what the ssh partition holds', async () => {
|
||||
const store = await createStore(diskAfterEarlierSession('pty-1'))
|
||||
|
||||
rendererPublishesPane(store, 'pty-1')
|
||||
expect(relayReattachBindsPane(store, 'pty-2')).toBe(true)
|
||||
// The other plane names the predecessor. It does not get a vote here — local speaking about
|
||||
// this leaf IS the desktop plane's live view, and letting the other copy outrank it is the
|
||||
// STA-3077 defect. Its binding to the loser is then scrubbed with every other reference to
|
||||
// that retired lease, so the two planes do not drift further apart.
|
||||
somethingRewritesTheSshPartition(store, 'pty-1')
|
||||
sshSpawnUpsertsLease(store, 'pty-2')
|
||||
|
||||
expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe(
|
||||
appPtyId('pty-2')
|
||||
)
|
||||
expect(liveLeaseIdsForPane(store)).toEqual(['pty-2'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077 step P: every production caller names that one home', () => {
|
||||
const summarize = (source: string): string => source.replace(/\s+/g, ' ').trim().slice(0, 90)
|
||||
|
||||
// A guard that behaves correctly is not evidence that both writers reach it,
|
||||
// and the defect IS that they disagree — so pin the call sites too.
|
||||
it('has no SSH pane binding write that selects the ssh partition', () => {
|
||||
const file = 'src/main/ipc/pty.ts'
|
||||
const calls = callArgumentsIn(readFileSync(file, 'utf-8'), 'persistPtyBinding')
|
||||
expect(calls.length, `${file} no longer writes pane bindings`).toBeGreaterThan(0)
|
||||
|
||||
const partitioned = calls
|
||||
.filter((call) => call.includes('toSshExecutionHostId'))
|
||||
.map((call) => `${file}: ${summarize(call)}`)
|
||||
|
||||
expect(partitioned).toEqual([])
|
||||
})
|
||||
|
||||
// STRENGTHENED, not relaxed. This clause used to require the relay to hold a `persistPtyBinding`
|
||||
// call of its own and merely forbid an ssh-partition argument on it. Step F removed that call:
|
||||
// the relay binds through the one `bindPaneShell` producer, which is what makes the superseded-
|
||||
// pane fence live on reattach. Requiring ZERO direct writes here is the stronger property — a
|
||||
// second bind producer is exactly the defect that let spawn and reattach disagree.
|
||||
it('has no pane binding write in the relay that bypasses the one bind producer', () => {
|
||||
const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8')
|
||||
|
||||
expect(callArgumentsIn(source, 'persistPtyBinding').map(summarize)).toEqual([])
|
||||
expect(source).toContain('bindPaneShell(')
|
||||
})
|
||||
|
||||
// The readers must land in the same place; one that still consults
|
||||
// `ssh:<target>` reinstates the disagreement from the other side.
|
||||
it('has no stable-pane owner reader that selects the ssh partition', () => {
|
||||
const source = readFileSync('src/main/ipc/pty.ts', 'utf-8')
|
||||
const start = source.indexOf('function resolvePersistedStablePaneOwner')
|
||||
const readers = source.slice(start, source.indexOf('type StablePaneSpawnContext'))
|
||||
expect(start, 'stable-pane owner readers moved').toBeGreaterThan(0)
|
||||
expect(readers).toContain('getWorkspaceSession(')
|
||||
|
||||
const partitioned = readers
|
||||
.split('\n')
|
||||
.filter((line) => line.includes('toSshExecutionHostId'))
|
||||
.map(summarize)
|
||||
|
||||
expect(partitioned).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -1,742 +0,0 @@
|
||||
/**
|
||||
* STA-3077 behavioral oracles: reconnecting an SSH workspace must not add panes
|
||||
* the user never opened, and must not accumulate remote shells.
|
||||
*
|
||||
* These assert observable behavior, not a mechanism, so they stay valid under
|
||||
* any implementation that fixes the defect. Each case names the root cause it
|
||||
* pins from the #12264 diagnosis.
|
||||
*
|
||||
* Reported symptom: relay PTY count went 2 -> 19 -> 20 across three reconnects.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { getDefaultPersistedState } from '../shared/constants'
|
||||
|
||||
const testState = { dir: '' }
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: { getPath: () => testState.dir },
|
||||
safeStorage: { isEncryptionAvailable: () => false }
|
||||
}))
|
||||
vi.mock('./telemetry/client', () => ({ track: vi.fn() }))
|
||||
vi.mock('./telemetry/cohort-classifier', () => ({ getCohortAtEmit: vi.fn() }))
|
||||
|
||||
const TARGET = 'ssh-target-1'
|
||||
const WORKTREE = 'repo-1:wt-1'
|
||||
const TAB = 'tab-1'
|
||||
/** Must be a real layout leaf UUID — the store drops any other spelling. */
|
||||
const LEAF = '3f1c9a2e-7b4d-4e1a-9c8f-2d5e6a7b8c90'
|
||||
|
||||
async function createStore(state: Record<string, unknown> = {}) {
|
||||
mkdirSync(testState.dir, { recursive: true })
|
||||
writeFileSync(
|
||||
join(testState.dir, 'orca-data.json'),
|
||||
JSON.stringify({ ...getDefaultPersistedState(testState.dir), ...state }),
|
||||
'utf-8'
|
||||
)
|
||||
vi.resetModules()
|
||||
const { Store, initDataPath } = await import('./persistence')
|
||||
initDataPath()
|
||||
return new Store()
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
testState.dir = mkdtempSync(join(tmpdir(), 'orca-sta3077-'))
|
||||
})
|
||||
|
||||
/** One pane's lease, as the reattach path records it. */
|
||||
function leaseFor(ptyId: string, updatedAt: number) {
|
||||
return {
|
||||
targetId: TARGET,
|
||||
ptyId,
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
state: 'attached' as const,
|
||||
updatedAt
|
||||
}
|
||||
}
|
||||
|
||||
function liveLeasesForPane(store: {
|
||||
getSshRemotePtyLeases: (targetId?: string) => readonly {
|
||||
ptyId: string
|
||||
tabId?: string
|
||||
leafId?: string
|
||||
state: string
|
||||
}[]
|
||||
}) {
|
||||
return store
|
||||
.getSshRemotePtyLeases(TARGET)
|
||||
.filter(
|
||||
(lease) =>
|
||||
lease.tabId === TAB &&
|
||||
lease.leafId === LEAF &&
|
||||
lease.state !== 'terminated' &&
|
||||
lease.state !== 'expired'
|
||||
)
|
||||
}
|
||||
|
||||
describe('STA-3077: one pane owns at most one live remote PTY lease', () => {
|
||||
// RC1: lease uniqueness keys on (targetId, ptyId) only, so a pane whose PTY id
|
||||
// changes leaves its predecessor behind with nothing to retire it.
|
||||
it('does not accumulate a second live lease when one pane re-leases a new PTY id', async () => {
|
||||
const store = await createStore()
|
||||
|
||||
store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1))
|
||||
store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2))
|
||||
|
||||
expect(liveLeasesForPane(store)).toHaveLength(1)
|
||||
})
|
||||
|
||||
// RC1: the reported 2 -> 19 -> 20 growth. Lease count must not scale with
|
||||
// reconnect count for a fixed set of panes.
|
||||
it('keeps live lease count flat across repeated reconnects of one pane', async () => {
|
||||
const store = await createStore()
|
||||
|
||||
for (let reconnect = 0; reconnect < 10; reconnect += 1) {
|
||||
store.upsertSshRemotePtyLease(leaseFor(`relay-pty-${reconnect}`, reconnect + 1))
|
||||
}
|
||||
|
||||
expect(liveLeasesForPane(store)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: superseding respects the durable pane binding', () => {
|
||||
// Spawn-shaped setup: the creating branches are intentionally allowed here.
|
||||
function bindPaneTo(store: Awaited<ReturnType<typeof createStore>>, ptyId: string) {
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId,
|
||||
incarnationId: `inc-${ptyId}`
|
||||
})
|
||||
}
|
||||
|
||||
it('scrubs the predecessor binding when the arriving lease is the bound one', async () => {
|
||||
const store = await createStore()
|
||||
bindPaneTo(store, 'relay-pty-a')
|
||||
store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1))
|
||||
bindPaneTo(store, 'relay-pty-b')
|
||||
store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2))
|
||||
|
||||
expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-b'])
|
||||
const layout = store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]
|
||||
expect(layout?.ptyIdsByLeafId?.[LEAF]).toBe('relay-pty-b')
|
||||
})
|
||||
|
||||
// Expiring the bound predecessor here would detach a live pane, so both stay
|
||||
// live and reattach arbitrates with the binding in hand.
|
||||
it('defers instead of expiring the lease the pane is bound to', async () => {
|
||||
const store = await createStore()
|
||||
bindPaneTo(store, 'relay-pty-a')
|
||||
store.upsertSshRemotePtyLease(leaseFor('relay-pty-a', 1))
|
||||
store.upsertSshRemotePtyLease(leaseFor('relay-pty-b', 2))
|
||||
|
||||
expect(
|
||||
liveLeasesForPane(store)
|
||||
.map((lease) => lease.ptyId)
|
||||
.sort()
|
||||
).toEqual(['relay-pty-a', 'relay-pty-b'])
|
||||
expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe(
|
||||
'relay-pty-a'
|
||||
)
|
||||
|
||||
// Arbitration keeps the bound lease, not the newer one.
|
||||
expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1)
|
||||
expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-a'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: existing duplicate leases are healed, not revived', () => {
|
||||
// Installs that predate pane-keyed supersession already carry the duplicates
|
||||
// this bug accumulated. Preventing new ones does not help them.
|
||||
it('retires every stale duplicate for a pane and keeps the newest', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: Array.from({ length: 20 }, (_, index) => ({
|
||||
...leaseFor(`relay-pty-${index}`, index + 1),
|
||||
createdAt: index + 1
|
||||
}))
|
||||
})
|
||||
|
||||
const retired = await store.supersedeDuplicatePaneLeases(TARGET)
|
||||
|
||||
expect(retired).toBe(19)
|
||||
expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-19'])
|
||||
})
|
||||
|
||||
// Recency alone would retire the lease the pane is actually bound to whenever a
|
||||
// newer unbound lease exists, detaching a live pane instead of healing it.
|
||||
it('keeps the durably bound lease even when an unbound one is newer', async () => {
|
||||
const store = await createStore()
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-bound',
|
||||
incarnationId: 'inc-bound'
|
||||
})
|
||||
store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-bound', 1), createdAt: 1 })
|
||||
// Arrives later but no pane is bound to it.
|
||||
store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-newer', 99), createdAt: 99 })
|
||||
|
||||
await store.supersedeDuplicatePaneLeases(TARGET)
|
||||
|
||||
expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-bound'])
|
||||
})
|
||||
|
||||
// A retirement that is not durable must not be believed: it would read as
|
||||
// retired in memory and attached on disk for the rest of the session.
|
||||
it('rolls the retirement back when the durable write fails', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
// The durable write is the async twin: retirement runs on reconnect, and the sync flush
|
||||
// fsyncs a multi-MB file from the main thread. Reaching for a private name is not ideal,
|
||||
// but the property under test is the rollback, and it must follow whichever writer the
|
||||
// retirement actually awaits.
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockRejectedValue(new Error('disk full'))
|
||||
|
||||
expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(0)
|
||||
expect(liveLeasesForPane(store)).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('does not chase store mutations after the retirement generation is durable', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
const internal = store as unknown as { enqueueWrite: () => Promise<void> }
|
||||
const enqueueWrite = internal.enqueueWrite.bind(store)
|
||||
let injectedMutation = false
|
||||
const enqueueSpy = vi.spyOn(internal, 'enqueueWrite').mockImplementation(async () => {
|
||||
await enqueueWrite()
|
||||
if (!injectedMutation) {
|
||||
injectedMutation = true
|
||||
store.updateUI({ sidebarWidth: 777 })
|
||||
}
|
||||
})
|
||||
|
||||
expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1)
|
||||
|
||||
expect(enqueueSpy).toHaveBeenCalledOnce()
|
||||
enqueueSpy.mockRestore()
|
||||
await store.flushPendingOrThrowAsync({ drainToStableGeneration: false })
|
||||
})
|
||||
|
||||
it('retries when a mutation invalidates the retirement write before rename', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
const internal = store as unknown as { writeToDiskAsync: () => Promise<void> }
|
||||
const writeToDiskAsync = internal.writeToDiskAsync.bind(store)
|
||||
let injectedMutation = false
|
||||
const writeSpy = vi.spyOn(internal, 'writeToDiskAsync').mockImplementation(async () => {
|
||||
const write = writeToDiskAsync()
|
||||
if (!injectedMutation) {
|
||||
injectedMutation = true
|
||||
store.updateUI({ sidebarWidth: 778 })
|
||||
}
|
||||
await write
|
||||
})
|
||||
|
||||
expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(1)
|
||||
|
||||
expect(writeSpy).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('does not undo a concurrent confirmation of the provisional expired state', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'expired')
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(
|
||||
store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state
|
||||
).toBe('expired')
|
||||
await store.flushPendingOrThrowAsync({ drainToStableGeneration: false })
|
||||
})
|
||||
|
||||
it('rolls back after a rejected concurrent attach request', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'attached')
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(
|
||||
store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state
|
||||
).toBe('attached')
|
||||
})
|
||||
|
||||
it('does not revive a provisional loser after a concurrent same-pane upsert', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
store.upsertSshRemotePtyLease({ ...leaseFor('relay-pty-c', 3), createdAt: 3 })
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(liveLeasesForPane(store).map((lease) => lease.ptyId)).toEqual(['relay-pty-c'])
|
||||
})
|
||||
|
||||
it('does not restore attached state over a concurrent detach', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
store.markSshRemotePtyLeasesForShutdown(TARGET, 'detached')
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(
|
||||
store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state
|
||||
).toBe('expired')
|
||||
expect(
|
||||
store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-b')?.state
|
||||
).toBe('detached')
|
||||
})
|
||||
|
||||
it('does not roll back concurrent session or lease changes after a failed write', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
store.markSshRemotePtyLease(TARGET, 'relay-pty-a', 'terminated')
|
||||
const concurrentTab = 'tab-concurrent'
|
||||
const concurrentLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d'
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: concurrentTab,
|
||||
leafId: concurrentLeaf,
|
||||
ptyId: 'relay-pty-concurrent',
|
||||
incarnationId: 'inc-concurrent'
|
||||
})
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(
|
||||
store.getSshRemotePtyLeases(TARGET).find((lease) => lease.ptyId === 'relay-pty-a')?.state
|
||||
).toBe('terminated')
|
||||
expect(
|
||||
store.getWorkspaceSession().terminalLayoutsByTabId?.[concurrentTab]?.ptyIdsByLeafId?.[
|
||||
concurrentLeaf
|
||||
]
|
||||
).toBe('relay-pty-concurrent')
|
||||
})
|
||||
|
||||
it('does not restore a binding for a lease removed during a failed write', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-a',
|
||||
incarnationId: 'inc-a'
|
||||
})
|
||||
delete store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
store.removeSshRemotePtyLease(TARGET, 'relay-pty-a')
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(store.getSshRemotePtyLeases(TARGET).some((lease) => lease.ptyId === 'relay-pty-a')).toBe(
|
||||
false
|
||||
)
|
||||
expect(
|
||||
store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)?.ptyId
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('restores a tab binding into the moved leaf after a concurrent session replacement', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-a',
|
||||
incarnationId: 'inc-a'
|
||||
})
|
||||
delete store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
expect(
|
||||
store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)?.ptyId
|
||||
).toBeNull()
|
||||
const movedTabId = 'tab-moved'
|
||||
const replacement = structuredClone(store.getWorkspaceSession())
|
||||
const movedTab = replacement.tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === TAB)
|
||||
if (!movedTab || !replacement.terminalLayoutsByTabId?.[TAB]) {
|
||||
throw new Error('expected source tab and layout')
|
||||
}
|
||||
movedTab.id = movedTabId
|
||||
replacement.terminalLayoutsByTabId[movedTabId] = replacement.terminalLayoutsByTabId[TAB]
|
||||
delete replacement.terminalLayoutsByTabId[TAB]
|
||||
store.setWorkspaceSession(replacement)
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(
|
||||
store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]?.find((tab) => tab.id === movedTabId)
|
||||
?.ptyId
|
||||
).toBe('relay-pty-a')
|
||||
await store.flushPendingOrThrowAsync({ drainToStableGeneration: false })
|
||||
})
|
||||
|
||||
it('does not restore a retired binding after the leaf is deleted', async () => {
|
||||
const otherTab = 'tab-other'
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2, tabId: otherTab }
|
||||
]
|
||||
})
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-a',
|
||||
incarnationId: 'inc-a'
|
||||
})
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: otherTab,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-b',
|
||||
incarnationId: 'inc-b'
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.supersedeDuplicatePaneLeases(TARGET)
|
||||
expect(store.getWorkspaceSession().terminalLayoutsByTabId?.[TAB]?.ptyIdsByLeafId?.[LEAF]).toBe(
|
||||
undefined
|
||||
)
|
||||
const replacement = structuredClone(store.getWorkspaceSession())
|
||||
replacement.tabsByWorktree![WORKTREE] = []
|
||||
replacement.terminalLayoutsByTabId = {}
|
||||
store.setWorkspaceSession(replacement)
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
expect(await retirement).toBe(0)
|
||||
expect(store.getWorkspaceSession().tabsByWorktree?.[WORKTREE]).toEqual([])
|
||||
expect(store.getWorkspaceSession().terminalLayoutsByTabId).toEqual({})
|
||||
})
|
||||
|
||||
it('leaves distinct panes alone', async () => {
|
||||
const otherLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d'
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2, leafId: otherLeaf }
|
||||
]
|
||||
})
|
||||
|
||||
expect(await store.supersedeDuplicatePaneLeases(TARGET)).toBe(0)
|
||||
expect(store.getSshRemotePtyLeases(TARGET).filter((l) => l.state === 'attached')).toHaveLength(
|
||||
2
|
||||
)
|
||||
})
|
||||
|
||||
// A lease is rebuilt field by field on load, so a field added to the type but not to the
|
||||
// normalizer is dropped on every boot — it would look right in memory and be gone on the next
|
||||
// start, which is how a fence ships silently permitting everything. A synthesized identity is
|
||||
// refused outright rather than carried: it is not stable across reconnects, so one persisted by
|
||||
// another build would later read as a different shell and strand a live pane.
|
||||
it('keeps a host-attested incarnation across a reload and refuses a synthesized one', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1, incarnationId: 'inc-host-a' },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2, incarnationId: 'legacy:23:0:relay-pty-b' }
|
||||
]
|
||||
})
|
||||
|
||||
const byPtyId = new Map(store.getSshRemotePtyLeases(TARGET).map((l) => [l.ptyId, l]))
|
||||
expect(byPtyId.get('relay-pty-a')?.incarnationId).toBe('inc-host-a')
|
||||
expect(byPtyId.get('relay-pty-b')?.incarnationId).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('SSH reconnect lease retirement batching', () => {
|
||||
it('keeps concurrent terminal state when the asynchronous write fails', async () => {
|
||||
const store = await createStore({
|
||||
sshRemotePtyLeases: [
|
||||
{ ...leaseFor('relay-pty-a', 1), createdAt: 1 },
|
||||
{ ...leaseFor('relay-pty-b', 2), createdAt: 2 }
|
||||
]
|
||||
})
|
||||
let rejectWrite: (error: Error) => void = () => {}
|
||||
vi.spyOn(
|
||||
store as unknown as { flushDurableStateOrThrowAsync: () => Promise<void> },
|
||||
'flushDurableStateOrThrowAsync'
|
||||
).mockImplementation(
|
||||
() =>
|
||||
new Promise((_, reject) => {
|
||||
rejectWrite = reject
|
||||
})
|
||||
)
|
||||
|
||||
const retirement = store.markSshRemotePtyLeasesTerminatedAsync(TARGET, ['relay-pty-a'])
|
||||
const concurrentTab = 'tab-concurrent'
|
||||
const concurrentLeaf = '8a2b4c6d-1e3f-4a5b-8c7d-9e0f1a2b3c4d'
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: concurrentTab,
|
||||
leafId: concurrentLeaf,
|
||||
ptyId: 'relay-pty-concurrent',
|
||||
incarnationId: 'inc-concurrent'
|
||||
})
|
||||
rejectWrite(new Error('disk full'))
|
||||
|
||||
await expect(retirement).rejects.toThrow('disk full')
|
||||
expect(
|
||||
store.getWorkspaceSession().terminalLayoutsByTabId?.[concurrentTab]?.ptyIdsByLeafId?.[
|
||||
concurrentLeaf
|
||||
]
|
||||
).toBe('relay-pty-concurrent')
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: reattach binds panes, it never creates them', () => {
|
||||
// RC3: persistPtyBinding has four creating branches (mint tab, mint root leaf,
|
||||
// split root and graft leaf, mint layout). They are load-bearing for spawn and
|
||||
// wrong for reattach, where the pane either exists or is gone for good.
|
||||
it('does not mint a tab for a pane that no longer exists', async () => {
|
||||
const store = await createStore()
|
||||
|
||||
store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-a',
|
||||
incarnationId: 'inc-a',
|
||||
mayCreate: false
|
||||
})
|
||||
|
||||
const session = store.getWorkspaceSession()
|
||||
expect(session.tabsByWorktree?.[WORKTREE] ?? []).toHaveLength(0)
|
||||
expect(session.terminalLayoutsByTabId?.[TAB]).toBeUndefined()
|
||||
})
|
||||
|
||||
// Unknown is not dead: failing to resolve a pane must not be reported as
|
||||
// success, and must not terminate anything.
|
||||
it('reports an unresolved reattach rather than silently succeeding', async () => {
|
||||
const store = await createStore()
|
||||
|
||||
const bound = store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-a',
|
||||
incarnationId: 'inc-a',
|
||||
mayCreate: false
|
||||
})
|
||||
|
||||
expect(bound).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: exact-binding compare-and-swap', () => {
|
||||
// Already correct at HEAD. Pinned so a fix cannot regress it: a stale renderer
|
||||
// replay must not overwrite a binding the host has since re-admitted.
|
||||
it('refuses a write whose expected binding no longer matches', async () => {
|
||||
const store = await createStore()
|
||||
|
||||
const stale = store.persistPtyBinding({
|
||||
worktreeId: WORKTREE,
|
||||
tabId: TAB,
|
||||
leafId: LEAF,
|
||||
ptyId: 'relay-pty-b',
|
||||
incarnationId: 'inc-b',
|
||||
expectedBinding: { ptyId: 'relay-pty-a', incarnationId: 'inc-a' }
|
||||
})
|
||||
|
||||
expect(stale).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077 step F: live-layout tab resolution is reattach-only', () => {
|
||||
// The lease's tabId is the frozen side on reattach, so the live layout outranks it there. On a
|
||||
// SPAWN the opposite holds: the caller's tabId is fresh truth and the persisted layout is the
|
||||
// stale side, because the renderer publishes membership on a debounce. Breaking a pane out into
|
||||
// a new tab and spawning into it inside that window would otherwise resolve back to the tab the
|
||||
// pane just left, writing the durable binding and the fence under one tab while the lease and
|
||||
// the runtime registration use the other — the split-coordinate defect step F exists to remove.
|
||||
it('resolves the live tab only on the reattach bind, never on a spawn', async () => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const relay = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8')
|
||||
const ipc = readFileSync('src/main/ipc/pty.ts', 'utf-8')
|
||||
|
||||
// The reattach holds a tabId frozen in a lease, so it must resolve the live one first.
|
||||
expect(relay).toContain('resolvePaneShellTabId(')
|
||||
// Resolving BEFORE the durable write is what keeps a thrown write from losing the answer and
|
||||
// registering the pane in the graph under the tab it left.
|
||||
expect(relay.indexOf('resolvePaneShellTabId(')).toBeLessThan(relay.indexOf('bindPaneShell({'))
|
||||
// Spawn callers hold the fresh tabId; none of them may consult the debounced layout.
|
||||
expect(ipc).toContain('bindPaneShell({')
|
||||
expect(ipc.split('bindPaneShell({').slice(1).join('')).not.toContain('resolvePaneShellTabId')
|
||||
})
|
||||
})
|
||||
|
||||
describe('STA-3077: the reattach path actually refuses to create', () => {
|
||||
// This is the oracle the store-level tests could not provide: `mayCreate`
|
||||
// existed and was correct, but no production caller passed it, so reattach
|
||||
// still grafted panes. Pin the wiring, not just the capability.
|
||||
// The reattach bind now goes through the one `bindPaneShell` producer, and it
|
||||
// still refuses to create.
|
||||
it('passes mayCreate:false from the SSH reattach binding write', async () => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8')
|
||||
const bindCall = source.slice(
|
||||
source.indexOf('restoreReattachedPtyRuntime'),
|
||||
source.indexOf('private async attachPtyWithRetry')
|
||||
)
|
||||
expect(bindCall).toContain('bindPaneShell')
|
||||
expect(bindCall).toContain('mayCreate: false')
|
||||
})
|
||||
|
||||
// Strengthened, not relaxed: counting guards against calls went vacuous once
|
||||
// the direct store calls disappeared, so pin both halves instead.
|
||||
it('has no production persistPtyBinding caller that can create during reattach', async () => {
|
||||
const { readFileSync } = await import('node:fs')
|
||||
const source = readFileSync('src/main/ssh/ssh-relay-session.ts', 'utf-8')
|
||||
// Every bind in the relay session is a reattach; none may grow topology.
|
||||
expect(source.split('persistPtyBinding(').length - 1).toBe(0)
|
||||
const binds = source.split('bindPaneShell(').length - 1
|
||||
expect(binds).toBeGreaterThan(0)
|
||||
// Each `bindPaneShell(` call site must carry its own `mayCreate: false`.
|
||||
const guardedBinds = source
|
||||
.split('bindPaneShell(')
|
||||
.slice(1)
|
||||
.filter((tail) => tail.slice(0, tail.indexOf('})')).includes('mayCreate: false')).length
|
||||
expect(guardedBinds).toBe(binds)
|
||||
})
|
||||
})
|
||||
@@ -69,7 +69,6 @@ vi.mock('../providers/ssh-git-provider', () => ({
|
||||
SshGitProvider: class MockSshGitProvider {}
|
||||
}))
|
||||
vi.mock('../ipc/pty', () => ({
|
||||
resolvePaneShellTabId: vi.fn(() => undefined),
|
||||
registerSshPtyProvider: vi.fn(),
|
||||
unregisterSshPtyProvider: vi.fn(),
|
||||
getSshPtyProvider: vi.fn(),
|
||||
@@ -187,24 +186,24 @@ describe('SshRelaySession abandoned remote PTYs', () => {
|
||||
expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID)
|
||||
})
|
||||
|
||||
// INVERTED for STA-3077 step E-0. This case previously asserted that a bare not-found retires the
|
||||
// lease and reports `pty:exit { code: -1 }`, on the premise that attach proves liveness before
|
||||
// answering not-found. It does not: the relay answers not-found for a pane-identity mismatch and
|
||||
// for any id it merely cannot hand back, so the premise licensed a fabricated death certificate.
|
||||
// The clause is kept — inverted — so the new intent stays covered rather than silently dropped.
|
||||
it('leaves the shell running when the relay only reports the PTY as not found', async () => {
|
||||
it('retires the lease without a kill when the relay proves the PTY is gone', async () => {
|
||||
// pty.attach verifies process liveness before answering not-found, so this is the one branch
|
||||
// with positive proof of death — and a dead process needs no shutdown request.
|
||||
const { deps, shutdown } = await establishWithFailingReattach(
|
||||
new Error('PTY "pty-live" not found')
|
||||
)
|
||||
|
||||
expect(shutdown).not.toHaveBeenCalled()
|
||||
expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith(
|
||||
expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'target-1',
|
||||
'pty-live',
|
||||
'expired'
|
||||
)
|
||||
expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID)
|
||||
expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything())
|
||||
expect(clearProviderPtyState).toHaveBeenCalledWith(APP_PTY_ID)
|
||||
expect(deps.mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', {
|
||||
id: APP_PTY_ID,
|
||||
code: -1
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps a recovered session attached when reattach succeeds after an earlier drop', async () => {
|
||||
|
||||
@@ -1,358 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { formatPtyExitedError } from '../../shared/ssh-pty-failure-tokens'
|
||||
import { SshRelaySession } from './ssh-relay-session'
|
||||
import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures'
|
||||
|
||||
const { acceptOutputExitMock, muxRequestMock } = vi.hoisted(() => ({
|
||||
acceptOutputExitMock: vi.fn().mockResolvedValue(undefined),
|
||||
muxRequestMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() }))
|
||||
vi.mock('./ssh-pty-consumer-session', () => ({
|
||||
openSshPtyConsumerSession: vi.fn(
|
||||
async (_mux: unknown, options: { clientInstanceId: string }) => ({
|
||||
clientInstanceId: options.clientInstanceId,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'test-owner-lease'
|
||||
})
|
||||
)
|
||||
}))
|
||||
vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({
|
||||
acceptSshPtyOutputData: vi.fn().mockResolvedValue(undefined),
|
||||
acceptSshPtyOutputExit: acceptOutputExitMock,
|
||||
allocateSshPtyProviderGeneration: vi.fn(() => 17),
|
||||
beginSshPtyOutputGenerationMigration: vi.fn(() => ({
|
||||
byPty: new Map(),
|
||||
completion: Promise.resolve()
|
||||
})),
|
||||
closeSshPtyOutputGeneration: vi.fn(),
|
||||
getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []),
|
||||
applySshPtySourceRecoveryCancellationProof: vi.fn(() => true),
|
||||
installSshPtySourceAckPublisher: vi.fn(() => () => {}),
|
||||
installSshPtySourceCancellationPublisher: vi.fn(() => () => {})
|
||||
}))
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') }))
|
||||
vi.mock('./ssh-remote-orca-cli', () => ({
|
||||
runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' })
|
||||
}))
|
||||
vi.mock('./ssh-channel-multiplexer', () => ({
|
||||
SshChannelMultiplexer: class MockSshChannelMultiplexer {
|
||||
notify = vi.fn()
|
||||
notifyWithSettlement = vi.fn()
|
||||
request = muxRequestMock
|
||||
onNotification = vi.fn().mockReturnValue(() => {})
|
||||
onNotificationByMethod = vi.fn().mockReturnValue(() => {})
|
||||
onRequest = vi.fn().mockReturnValue(() => {})
|
||||
onDispose = vi.fn().mockReturnValue(() => {})
|
||||
dispose = vi.fn()
|
||||
isDisposed = vi.fn().mockReturnValue(false)
|
||||
}
|
||||
}))
|
||||
vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({
|
||||
installRemoteManagedAgentHooks: vi.fn().mockResolvedValue([])
|
||||
}))
|
||||
vi.mock('../providers/ssh-pty-provider', () => ({
|
||||
SshPtyProvider: class MockSshPtyProvider {
|
||||
onData = vi.fn().mockReturnValue(() => {})
|
||||
onReplay = vi.fn().mockReturnValue(() => {})
|
||||
onExit = vi.fn().mockReturnValue(() => {})
|
||||
attach = vi.fn().mockResolvedValue(undefined)
|
||||
attachForReconnect = vi.fn().mockResolvedValue({})
|
||||
dispose = vi.fn()
|
||||
}
|
||||
}))
|
||||
vi.mock('../providers/ssh-filesystem-provider', () => ({
|
||||
SshFilesystemProvider: class MockSshFilesystemProvider {
|
||||
dispose = vi.fn()
|
||||
}
|
||||
}))
|
||||
vi.mock('../providers/ssh-git-provider', () => ({
|
||||
SshGitProvider: class MockSshGitProvider {}
|
||||
}))
|
||||
vi.mock('../ipc/pty', () => ({
|
||||
resolvePaneShellTabId: vi.fn(() => undefined),
|
||||
registerSshPtyProvider: vi.fn(),
|
||||
unregisterSshPtyProvider: vi.fn(),
|
||||
getSshPtyProvider: vi.fn(),
|
||||
getPtyIdsForConnection: vi.fn().mockReturnValue([]),
|
||||
clearPtyOwnershipForConnection: vi.fn(),
|
||||
clearProviderPtyState: vi.fn(),
|
||||
deletePtyOwnership: vi.fn(),
|
||||
setPtyOwnership: vi.fn(),
|
||||
restorePtyIncarnation: vi.fn(),
|
||||
isCurrentPtyExit: vi.fn(() => true),
|
||||
answerStartupTerminalColorQueriesForPty: vi.fn((_id: string, data: string) => data)
|
||||
}))
|
||||
vi.mock('../providers/ssh-filesystem-dispatch', () => ({
|
||||
registerSshFilesystemProvider: vi.fn(),
|
||||
unregisterSshFilesystemProvider: vi.fn(),
|
||||
getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() })
|
||||
}))
|
||||
vi.mock('../providers/ssh-git-dispatch', () => ({
|
||||
registerSshGitProvider: vi.fn(),
|
||||
unregisterSshGitProvider: vi.fn()
|
||||
}))
|
||||
|
||||
const {
|
||||
registerSshPtyProvider,
|
||||
getSshPtyProvider,
|
||||
getPtyIdsForConnection,
|
||||
clearProviderPtyState,
|
||||
deletePtyOwnership
|
||||
} = await import('../ipc/pty')
|
||||
|
||||
const TARGET_ID = 'target-1'
|
||||
const RELAY_PTY_ID = 'pty-live'
|
||||
const APP_PTY_ID = 'ssh:target-1@@pty-live'
|
||||
const LEAF_ID = '11111111-1111-4111-8111-111111111111'
|
||||
const PROVIDER_GENERATION = 17
|
||||
|
||||
// The relay's own not-found wording; `isSshPtyNotFoundError` matches it and nothing narrows it
|
||||
// further, so it arrives carrying no evidence about whether the shell is alive.
|
||||
const UNPROVEN_NOT_FOUND = `PTY "${RELAY_PTY_ID}" not found`
|
||||
|
||||
function detachedLease() {
|
||||
return {
|
||||
targetId: TARGET_ID,
|
||||
ptyId: RELAY_PTY_ID,
|
||||
state: 'detached' as const,
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: LEAF_ID,
|
||||
incarnationId: 'inc-proven-exit'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* STA-3077 / design E-0: a reattach that ends in a bare not-found tells us the relay cannot hand
|
||||
* this id back — never that the process died. Synthesizing `pty:exit { code: -1 }` from it is a
|
||||
* fabricated death certificate: the renderer treats it as a real exit and the surviving remote
|
||||
* shell (and any agent inside it) is written off. Only an exit the relay actually observed may
|
||||
* report an exit; everything else routes into the non-destructive detached branch.
|
||||
*/
|
||||
describe('SshRelaySession reattach failures never fabricate an exit', () => {
|
||||
let warn: ReturnType<typeof vi.spyOn>
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
muxRequestMock.mockReset()
|
||||
muxRequestMock.mockResolvedValue([])
|
||||
acceptOutputExitMock.mockResolvedValue(undefined)
|
||||
mockDeploySuccess()
|
||||
vi.mocked(getPtyIdsForConnection).mockReturnValue([])
|
||||
warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'info').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
/** Drives the real connect-path reattach loop, so the failure reaches the handler as production does. */
|
||||
async function reattachFailsWith(message: string): Promise<{
|
||||
deps: ReturnType<typeof createMockDeps>
|
||||
attachForReconnect: ReturnType<typeof vi.fn>
|
||||
}> {
|
||||
const deps = createMockDeps()
|
||||
const attachForReconnect = vi.fn().mockRejectedValue(new Error(message))
|
||||
vi.mocked(getSshPtyProvider).mockReturnValue({
|
||||
attachForReconnect,
|
||||
shutdown: vi.fn().mockResolvedValue(undefined),
|
||||
dispose: vi.fn()
|
||||
} as unknown as ReturnType<typeof getSshPtyProvider>)
|
||||
vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([detachedLease()] as ReturnType<
|
||||
typeof deps.mockStore.getSshRemotePtyLeases
|
||||
>)
|
||||
const session = new SshRelaySession(
|
||||
TARGET_ID,
|
||||
deps.getMainWindow,
|
||||
deps.mockStore,
|
||||
deps.mockPortForward
|
||||
)
|
||||
|
||||
await session.establish(deps.mockConn)
|
||||
// Precondition, not a claim: if the reattach never ran, every "did not destroy" clause below
|
||||
// would pass for the wrong reason.
|
||||
expect(attachForReconnect).toHaveBeenCalled()
|
||||
|
||||
return { deps, attachForReconnect }
|
||||
}
|
||||
|
||||
function warnings(): string[] {
|
||||
return warn.mock.calls.map((call) => String(call[0]))
|
||||
}
|
||||
|
||||
it('reaches the failure handler through the real reattach path', async () => {
|
||||
// Producer pin: without this the remaining clauses could pass on a path production never takes.
|
||||
const { attachForReconnect } = await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(attachForReconnect.mock.calls[0]?.[0]).toBe(RELAY_PTY_ID)
|
||||
})
|
||||
|
||||
it('sends no synthetic pty:exit to the pane on an unproven not-found', async () => {
|
||||
const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything())
|
||||
})
|
||||
|
||||
it('keeps pty ownership on an unproven not-found', async () => {
|
||||
await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(deletePtyOwnership).not.toHaveBeenCalledWith(APP_PTY_ID)
|
||||
})
|
||||
|
||||
it('keeps provider pty state on an unproven not-found', async () => {
|
||||
await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(clearProviderPtyState).not.toHaveBeenCalledWith(APP_PTY_ID)
|
||||
})
|
||||
|
||||
it('leaves the lease terminable instead of expiring it on an unproven not-found', async () => {
|
||||
const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith(
|
||||
TARGET_ID,
|
||||
RELAY_PTY_ID,
|
||||
'expired'
|
||||
)
|
||||
})
|
||||
|
||||
// The counterpart. An exit the relay watched settles it, so the record is retired rather than
|
||||
// re-attached on every future reconnect for the life of the install. `terminated`, not `expired`:
|
||||
// expiry is the state the recovery grant reads, and retiring a record must not also authorise a
|
||||
// replacement.
|
||||
it('retires the lease on an exit the relay proved', async () => {
|
||||
const { deps } = await reattachFailsWith(
|
||||
formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-proven-exit')
|
||||
)
|
||||
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledWith(TARGET_ID, [
|
||||
RELAY_PTY_ID
|
||||
])
|
||||
})
|
||||
|
||||
it('persists a reconnect batch of proven exits in one async write', async () => {
|
||||
const deps = createMockDeps()
|
||||
const secondPtyId = 'pty-second'
|
||||
const secondIncarnationId = 'inc-proven-second-exit'
|
||||
const attachForReconnect = vi.fn().mockImplementation((ptyId: string) => {
|
||||
const incarnationId = ptyId === RELAY_PTY_ID ? 'inc-proven-exit' : secondIncarnationId
|
||||
return Promise.reject(new Error(formatPtyExitedError(ptyId, 0, incarnationId)))
|
||||
})
|
||||
vi.mocked(getSshPtyProvider).mockReturnValue({
|
||||
attachForReconnect,
|
||||
shutdown: vi.fn().mockResolvedValue(undefined),
|
||||
dispose: vi.fn()
|
||||
} as unknown as ReturnType<typeof getSshPtyProvider>)
|
||||
vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([
|
||||
detachedLease(),
|
||||
{
|
||||
...detachedLease(),
|
||||
ptyId: secondPtyId,
|
||||
leafId: '22222222-2222-4222-8222-222222222222',
|
||||
incarnationId: secondIncarnationId
|
||||
}
|
||||
] as ReturnType<typeof deps.mockStore.getSshRemotePtyLeases>)
|
||||
const session = new SshRelaySession(
|
||||
TARGET_ID,
|
||||
deps.getMainWindow,
|
||||
deps.mockStore,
|
||||
deps.mockPortForward
|
||||
)
|
||||
|
||||
await session.establish(deps.mockConn)
|
||||
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledOnce()
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).toHaveBeenCalledWith(TARGET_ID, [
|
||||
RELAY_PTY_ID,
|
||||
secondPtyId
|
||||
])
|
||||
expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not retire the lease for proof naming another PTY', async () => {
|
||||
const { deps } = await reattachFailsWith(
|
||||
formatPtyExitedError('pty-some-other-shell', 0, 'inc-proven-exit')
|
||||
)
|
||||
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not retire the lease for proof naming another incarnation', async () => {
|
||||
const { deps } = await reattachFailsWith(
|
||||
formatPtyExitedError(RELAY_PTY_ID, 0, 'inc-some-other-shell')
|
||||
)
|
||||
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not retire the lease when the failure proves nothing', async () => {
|
||||
const { deps } = await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesTerminatedAsync).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('routes an unproven not-found into the non-destructive detached branch', async () => {
|
||||
// That branch announces itself with the "leaving detached" warning; the destructive one says
|
||||
// "dropping stale". The log is how the two are told apart from outside the session.
|
||||
await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(warnings().some((line) => line.includes(RELAY_PTY_ID) && /detached/i.test(line))).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
it('does not take the drop-stale branch on an unproven not-found', async () => {
|
||||
await reattachFailsWith(UNPROVEN_NOT_FOUND)
|
||||
|
||||
expect(warnings().some((line) => /Dropping stale PTY/i.test(line))).toBe(false)
|
||||
})
|
||||
|
||||
it('still reports an exit the relay actually observed', async () => {
|
||||
// Contrast: the only evidence that authorizes a death claim is an exit the relay saw on a live
|
||||
// stream. Removing the fabricated one must not mute this one.
|
||||
const deps = createMockDeps()
|
||||
vi.mocked(getSshPtyProvider).mockReturnValue({
|
||||
dispose: vi.fn()
|
||||
} as unknown as ReturnType<typeof getSshPtyProvider>)
|
||||
vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue(
|
||||
[] as ReturnType<typeof deps.mockStore.getSshRemotePtyLeases>
|
||||
)
|
||||
const session = new SshRelaySession(
|
||||
TARGET_ID,
|
||||
deps.getMainWindow,
|
||||
deps.mockStore,
|
||||
deps.mockPortForward
|
||||
)
|
||||
await session.establish(deps.mockConn)
|
||||
const provider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as {
|
||||
onExit: ReturnType<typeof vi.fn>
|
||||
}
|
||||
const onExit = provider.onExit.mock.calls[0]?.[0] as (payload: unknown) => void
|
||||
|
||||
onExit({
|
||||
id: APP_PTY_ID,
|
||||
code: 3,
|
||||
incarnationId: 'incarnation-1',
|
||||
providerGeneration: PROVIDER_GENERATION,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(acceptOutputExitMock).toHaveBeenCalledWith({
|
||||
id: APP_PTY_ID,
|
||||
code: 3,
|
||||
providerGeneration: PROVIDER_GENERATION,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
)
|
||||
expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
TARGET_ID,
|
||||
RELAY_PTY_ID,
|
||||
'terminated'
|
||||
)
|
||||
expect(deletePtyOwnership).toHaveBeenCalledWith(APP_PTY_ID)
|
||||
})
|
||||
})
|
||||
@@ -156,7 +156,6 @@ function createSession(targetId: string): InstanceType<typeof SshRelaySession> {
|
||||
upsertSshPtyConsumerRecovery: vi.fn(),
|
||||
removeSshPtyConsumerRecovery: vi.fn(),
|
||||
getSshRemotePtyLeases: vi.fn().mockReturnValue([]),
|
||||
supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0),
|
||||
markSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLeases: vi.fn(),
|
||||
markSshRemotePtyLeasesAsync: vi.fn(),
|
||||
|
||||
@@ -403,6 +403,7 @@ describe('SshRelaySession data delivery', () => {
|
||||
expect(retryCalls[0]).toHaveProperty('resume')
|
||||
expect(attachForReconnectMock).toHaveBeenCalledWith(
|
||||
'pty-1',
|
||||
undefined,
|
||||
Object.freeze({ status: 'checkpointUnavailable' })
|
||||
)
|
||||
second.dispose()
|
||||
@@ -761,6 +762,7 @@ describe('SshRelaySession data delivery', () => {
|
||||
|
||||
expect(attachForReconnectMock).toHaveBeenCalledWith(
|
||||
'pty-1',
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
status: 'checkpoint',
|
||||
deliveryToken: 'old-token',
|
||||
|
||||
@@ -199,6 +199,7 @@ describe('SshRelaySession model migration', () => {
|
||||
|
||||
expect(attachForReconnectMock).toHaveBeenCalledWith(
|
||||
'pty-1',
|
||||
undefined,
|
||||
Object.freeze({
|
||||
status: 'checkpoint',
|
||||
clientGeneration: 1,
|
||||
@@ -258,6 +259,7 @@ describe('SshRelaySession model migration', () => {
|
||||
|
||||
expect(attachForReconnectMock).toHaveBeenCalledWith(
|
||||
'pty-1',
|
||||
undefined,
|
||||
Object.freeze({ status: 'checkpointUnavailable' })
|
||||
)
|
||||
})
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
/**
|
||||
* The superseded-PTY fence must be LIVE on the relay reattach path — the path it
|
||||
* was built for. Production wiring only: the real `pty:spawn`/`pty:write`
|
||||
* handlers from src/main/ipc/pty.ts and a real `SshRelaySession` whose reconnect
|
||||
* drives `restoreReattachedPtyRuntime`; only the wire below the provider is fake.
|
||||
*
|
||||
* Today reattach binds the pane through `runtime.registerPty` while spawn binds
|
||||
* through `rememberPaneKeyForPty`, so the fence maps never learn the successor
|
||||
* and a keystroke queued for the superseded PTY still reaches the host.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { makePaneKey } from '../../shared/stable-pane-id'
|
||||
|
||||
const handlers = new Map<string, (event: unknown, args: unknown) => unknown>()
|
||||
const listeners = new Map<string, (event: unknown, args: unknown) => unknown>()
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
BrowserWindow: undefined,
|
||||
app: {
|
||||
isPackaged: true,
|
||||
getPath: () => '/tmp/orca-reattach-fence-test',
|
||||
getVersion: () => '0.0.0-test'
|
||||
},
|
||||
powerMonitor: { on: vi.fn() },
|
||||
nativeTheme: { shouldUseDarkColors: true },
|
||||
ipcMain: {
|
||||
handle: (channel: string, handler: (event: unknown, args: unknown) => unknown) => {
|
||||
handlers.set(channel, handler)
|
||||
},
|
||||
on: (channel: string, listener: (event: unknown, args: unknown) => unknown) => {
|
||||
listeners.set(channel, listener)
|
||||
},
|
||||
removeHandler: (channel: string) => handlers.delete(channel),
|
||||
removeAllListeners: (channel: string) => listeners.delete(channel)
|
||||
}
|
||||
}))
|
||||
|
||||
const { providerInstances, spawnedPtyIds, attachIncarnationId } = vi.hoisted(() => ({
|
||||
providerInstances: [] as unknown[],
|
||||
spawnedPtyIds: [] as string[],
|
||||
attachIncarnationId: 'incarnation-reattached'
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() }))
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') }))
|
||||
vi.mock('./ssh-remote-orca-cli', () => ({
|
||||
runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' })
|
||||
}))
|
||||
vi.mock('./ssh-pty-consumer-session', () => ({
|
||||
openSshPtyConsumerSession: vi.fn(
|
||||
async (_mux: unknown, options: { clientInstanceId: string }) => ({
|
||||
clientInstanceId: options.clientInstanceId,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'test-owner-lease'
|
||||
})
|
||||
)
|
||||
}))
|
||||
vi.mock('./ssh-channel-multiplexer', () => ({
|
||||
SshChannelMultiplexer: class MockSshChannelMultiplexer {
|
||||
notify = vi.fn()
|
||||
notifyWithSettlement = vi.fn()
|
||||
request = vi.fn().mockResolvedValue([])
|
||||
onNotification = vi.fn().mockReturnValue(() => {})
|
||||
onNotificationByMethod = vi.fn().mockReturnValue(() => {})
|
||||
onRequest = vi.fn().mockReturnValue(() => {})
|
||||
onDispose = vi.fn().mockReturnValue(() => {})
|
||||
dispose = vi.fn()
|
||||
isDisposed = vi.fn().mockReturnValue(false)
|
||||
}
|
||||
}))
|
||||
vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({
|
||||
installRemoteManagedAgentHooks: vi.fn()
|
||||
}))
|
||||
// The relay session constructs and registers this provider itself, so the real
|
||||
// pty.ts registry hands the same instance to pty:spawn and pty:write.
|
||||
vi.mock('../providers/ssh-pty-provider', () => ({
|
||||
isSshPtyNotFoundError: (error: unknown) => String(error).includes('not found'),
|
||||
isSshPtyIdentityMismatchError: (error: unknown) => String(error).includes('identity mismatch'),
|
||||
SshPtyProvider: class MockSshPtyProvider {
|
||||
spawn = vi.fn(async () => ({ id: spawnedPtyIds.shift() ?? 'ssh:target-1@@pty-unexpected' }))
|
||||
write = vi.fn()
|
||||
resize = vi.fn()
|
||||
shutdown = vi.fn()
|
||||
sendSignal = vi.fn()
|
||||
getCwd = vi.fn()
|
||||
getInitialCwd = vi.fn()
|
||||
clearBuffer = vi.fn()
|
||||
acknowledgeDataEvent = vi.fn()
|
||||
hasChildProcesses = vi.fn()
|
||||
getForegroundProcess = vi.fn()
|
||||
serialize = vi.fn()
|
||||
revive = vi.fn()
|
||||
listProcesses = vi.fn(async () => [])
|
||||
getDefaultShell = vi.fn()
|
||||
getProfiles = vi.fn()
|
||||
onData = vi.fn().mockReturnValue(() => {})
|
||||
onReplay = vi.fn().mockReturnValue(() => {})
|
||||
onExit = vi.fn().mockReturnValue(() => {})
|
||||
attach = vi.fn().mockResolvedValue(undefined)
|
||||
attachForReconnect = vi.fn().mockResolvedValue({ incarnationId: attachIncarnationId })
|
||||
dispose = vi.fn()
|
||||
|
||||
constructor() {
|
||||
providerInstances.push(this)
|
||||
}
|
||||
}
|
||||
}))
|
||||
vi.mock('../providers/ssh-filesystem-provider', () => ({
|
||||
SshFilesystemProvider: class MockSshFilesystemProvider {
|
||||
dispose = vi.fn()
|
||||
}
|
||||
}))
|
||||
vi.mock('../providers/ssh-git-provider', () => ({
|
||||
SshGitProvider: class MockSshGitProvider {}
|
||||
}))
|
||||
vi.mock('../providers/ssh-filesystem-dispatch', () => ({
|
||||
registerSshFilesystemProvider: vi.fn(),
|
||||
unregisterSshFilesystemProvider: vi.fn(),
|
||||
getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() })
|
||||
}))
|
||||
vi.mock('../providers/ssh-git-dispatch', () => ({
|
||||
registerSshGitProvider: vi.fn(),
|
||||
unregisterSshGitProvider: vi.fn()
|
||||
}))
|
||||
|
||||
const { SshRelaySession } = await import('./ssh-relay-session')
|
||||
const { deployAndLaunchRelay } = await import('./ssh-relay-deploy')
|
||||
const {
|
||||
registerPtyHandlers,
|
||||
unregisterSshPtyProvider,
|
||||
getPtyIdForPaneKey,
|
||||
getPtyIdsForConnection,
|
||||
deletePtyOwnership
|
||||
} = await import('../ipc/pty')
|
||||
|
||||
const TARGET = 'target-1'
|
||||
const TAB_ID = 'tab-1'
|
||||
|
||||
type ProviderMock = { write: ReturnType<typeof vi.fn> }
|
||||
|
||||
function makeMainWindow() {
|
||||
const webContents = {
|
||||
id: 1,
|
||||
send: vi.fn(),
|
||||
on: vi.fn(),
|
||||
once: vi.fn(),
|
||||
removeListener: vi.fn(),
|
||||
isDestroyed: () => false
|
||||
}
|
||||
return { isDestroyed: () => false, isVisible: () => true, isMinimized: () => false, webContents }
|
||||
}
|
||||
|
||||
/** A live layout in which `leafId` sits in `tabId` — the tab the pane is in RIGHT NOW.
|
||||
* The tab must also exist in `tabsByWorktree`: resolution ignores a layout whose tab is gone, so
|
||||
* a fixture without one would resolve nothing and the clause would pass for the wrong reason. */
|
||||
function sessionWithLeafInTab(tabId: string, leafId: string) {
|
||||
return {
|
||||
tabsByWorktree: { 'worktree-1': [{ id: tabId, worktreeId: 'worktree-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
[tabId]: { root: { type: 'leaf' as const, leafId }, ptyIdsByLeafId: {} }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function makeStore(leases: unknown[], session?: ReturnType<typeof sessionWithLeafInTab>) {
|
||||
return {
|
||||
...(session ? { getWorkspaceSession: vi.fn(() => session) } : {}),
|
||||
getRepos: vi.fn().mockReturnValue([]),
|
||||
getSshPtyConsumerRecovery: vi.fn().mockReturnValue(null),
|
||||
upsertSshPtyConsumerRecovery: vi.fn(),
|
||||
removeSshPtyConsumerRecovery: vi.fn(),
|
||||
getSshRemotePtyLeases: vi.fn(() => leases),
|
||||
supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0),
|
||||
markSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLeases: vi.fn(),
|
||||
markSshRemotePtyLeasesAsync: vi.fn(),
|
||||
markSshRemotePtyLeasesForShutdown: vi.fn(),
|
||||
markSshRemotePtyLeasesAttachedAsync: vi.fn(),
|
||||
upsertSshRemotePtyLease: vi.fn(),
|
||||
persistPtyBinding: vi.fn().mockReturnValue(true)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One pane: spawned over SSH, then rebound by a reconnect whose durable lease
|
||||
* names a different remote PTY — the STA-3077 shape.
|
||||
*/
|
||||
async function spawnThenReattachPane(leafId: string, currentTabId?: string) {
|
||||
const mainWindow = makeMainWindow()
|
||||
const predecessorPtyId = `ssh:${TARGET}@@pty-old-${leafId.slice(0, 4)}`
|
||||
const reattachedRelayPtyId = `pty-new-${leafId.slice(0, 4)}`
|
||||
const leases: unknown[] = []
|
||||
const store = makeStore(
|
||||
leases,
|
||||
currentTabId ? sessionWithLeafInTab(currentTabId, leafId) : undefined
|
||||
)
|
||||
const runtime = { onPtySpawned: vi.fn(), registerPty: vi.fn() }
|
||||
const session = new SshRelaySession(
|
||||
TARGET,
|
||||
vi.fn().mockReturnValue(mainWindow) as never,
|
||||
store as never,
|
||||
{ removeAllForwards: vi.fn() } as never,
|
||||
runtime as never
|
||||
)
|
||||
|
||||
registerPtyHandlers(
|
||||
mainWindow as never,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
store as never
|
||||
)
|
||||
await session.establish({} as never)
|
||||
|
||||
spawnedPtyIds.push(predecessorPtyId)
|
||||
await handlers.get('pty:spawn')!(null, {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
env: { ORCA_PANE_KEY: makePaneKey(TAB_ID, leafId) },
|
||||
connectionId: TARGET,
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: TAB_ID,
|
||||
leafId
|
||||
})
|
||||
|
||||
// The relay restarted; the pane's durable lease now names a different shell.
|
||||
leases.push({
|
||||
targetId: TARGET,
|
||||
ptyId: reattachedRelayPtyId,
|
||||
state: 'detached' as const,
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: TAB_ID,
|
||||
leafId
|
||||
})
|
||||
await session.reconnect({} as never)
|
||||
// Reconnect builds a fresh provider; writes route through the live one.
|
||||
const provider = providerInstances.at(-1) as ProviderMock
|
||||
|
||||
const write = (id: string): void => {
|
||||
listeners.get('pty:write')!({ sender: mainWindow.webContents }, { id, data: 'x' })
|
||||
}
|
||||
return {
|
||||
runtime,
|
||||
paneKey: makePaneKey(currentTabId ?? TAB_ID, leafId),
|
||||
predecessorPtyId,
|
||||
reattachedPtyId: `ssh:${TARGET}@@${reattachedRelayPtyId}`,
|
||||
provider,
|
||||
write,
|
||||
dispose: () => {
|
||||
deletePtyOwnership(predecessorPtyId)
|
||||
deletePtyOwnership(`ssh:${TARGET}@@${reattachedRelayPtyId}`)
|
||||
unregisterSshPtyProvider(TARGET)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('a relay reattach binds the pane through the same producer as spawn', () => {
|
||||
let cleanup: (() => void) | undefined
|
||||
|
||||
beforeEach(() => {
|
||||
handlers.clear()
|
||||
listeners.clear()
|
||||
providerInstances.length = 0
|
||||
spawnedPtyIds.length = 0
|
||||
vi.mocked(deployAndLaunchRelay).mockResolvedValue({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
platform: 'linux-x64'
|
||||
} as never)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
cleanup?.()
|
||||
cleanup = undefined
|
||||
})
|
||||
|
||||
it('refuses a queued write to the PTY the reattach superseded', async () => {
|
||||
const pane = await spawnThenReattachPane('11111111-1111-4111-8111-111111111111')
|
||||
cleanup = pane.dispose
|
||||
pane.provider.write.mockClear()
|
||||
// Still an owned, routable PTY — so only the fence can refuse the write.
|
||||
expect(getPtyIdsForConnection(TARGET)).toContain(pane.predecessorPtyId)
|
||||
|
||||
// A keystroke the renderer queued before the reconnect, still addressed to
|
||||
// the shell the pane no longer owns.
|
||||
pane.write(pane.predecessorPtyId)
|
||||
await Promise.resolve()
|
||||
|
||||
expect(pane.provider.write).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('resolves the pane to the reattached PTY', async () => {
|
||||
const pane = await spawnThenReattachPane('22222222-2222-4222-8222-222222222222')
|
||||
cleanup = pane.dispose
|
||||
|
||||
expect(getPtyIdForPaneKey(pane.paneKey)).toBe(pane.reattachedPtyId)
|
||||
})
|
||||
|
||||
// The lease's tabId is frozen at write time, but `detachTerminalPaneToTab` moves a live pane
|
||||
// into a new tab without killing its PTY. A producer that forwarded `lease.tabId` would key the
|
||||
// fence to the tab the pane LEFT — and every other clause here uses one tab on both sides, so
|
||||
// nothing else in this file would notice.
|
||||
it('keys the pane to the tab the leaf is in now, not the one its lease was written in', async () => {
|
||||
const leafId = '44444444-4444-4444-8444-444444444444'
|
||||
const pane = await spawnThenReattachPane(leafId, 'tab-moved-to')
|
||||
cleanup = pane.dispose
|
||||
|
||||
expect(pane.paneKey).toBe(makePaneKey('tab-moved-to', leafId))
|
||||
expect(getPtyIdForPaneKey(pane.paneKey)).toBe(pane.reattachedPtyId)
|
||||
expect(getPtyIdForPaneKey(makePaneKey(TAB_ID, leafId))).not.toBe(pane.reattachedPtyId)
|
||||
// The runtime graph must land on the SAME tab as the record and the fence. Registering under
|
||||
// the lease's frozen tabId would split the pane across two tabs and ensure a mobile surface
|
||||
// for the one it left.
|
||||
expect(pane.runtime.registerPty).toHaveBeenCalledWith(
|
||||
pane.reattachedPtyId,
|
||||
'worktree-1',
|
||||
TARGET,
|
||||
expect.objectContaining({ tabId: 'tab-moved-to', leafId })
|
||||
)
|
||||
})
|
||||
|
||||
// Guards the clause above from a fence that simply refuses everything.
|
||||
it('still delivers a write to the reattached PTY', async () => {
|
||||
const pane = await spawnThenReattachPane('33333333-3333-4333-8333-333333333333')
|
||||
cleanup = pane.dispose
|
||||
pane.provider.write.mockClear()
|
||||
|
||||
pane.write(pane.reattachedPtyId)
|
||||
await Promise.resolve()
|
||||
|
||||
expect(pane.provider.write).toHaveBeenCalledWith(pane.reattachedPtyId, 'x')
|
||||
})
|
||||
})
|
||||
@@ -101,9 +101,6 @@ vi.mock('../providers/ssh-git-provider', () => ({
|
||||
SshGitProvider: class MockSshGitProvider {}
|
||||
}))
|
||||
vi.mock('../ipc/pty', () => ({
|
||||
resolvePaneShellTabId: vi.fn(() => undefined),
|
||||
// Step F: the single pane->shell bind producer the relay now calls.
|
||||
bindPaneShell: vi.fn(() => true),
|
||||
registerSshPtyProvider: vi.fn(),
|
||||
unregisterSshPtyProvider: vi.fn(),
|
||||
getSshPtyProvider: vi.fn(),
|
||||
@@ -127,7 +124,6 @@ vi.mock('../providers/ssh-git-dispatch', () => ({
|
||||
}))
|
||||
|
||||
const {
|
||||
bindPaneShell,
|
||||
registerSshPtyProvider,
|
||||
getSshPtyProvider,
|
||||
getPtyIdsForConnection,
|
||||
@@ -439,43 +435,18 @@ describe('SshRelaySession reconnect incarnation ordering', () => {
|
||||
incarnationId
|
||||
})
|
||||
expect(runtime.onPtySpawned).not.toHaveBeenCalled()
|
||||
// Step F moved the observation point to the one bind producer; the intent is
|
||||
// unchanged — exact incarnation proof, and mayCreate:false so reattach binds
|
||||
// an existing pane and never grafts one back.
|
||||
expect(bindPaneShell).toHaveBeenCalledWith({
|
||||
store: expect.anything(),
|
||||
expect(mockStore.persistPtyBinding).toHaveBeenCalledWith({
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: INCARNATION_LEAF_ID,
|
||||
ptyId: APP_PTY_ID,
|
||||
incarnationId,
|
||||
mayCreate: false
|
||||
incarnationId
|
||||
})
|
||||
expect(vi.mocked(bindPaneShell).mock.invocationCallOrder[0]).toBeLessThan(
|
||||
expect(vi.mocked(mockStore.persistPtyBinding).mock.invocationCallOrder[0]).toBeLessThan(
|
||||
vi.mocked(mockStore.markSshRemotePtyLeasesAttachedAsync).mock.invocationCallOrder[0]!
|
||||
)
|
||||
})
|
||||
|
||||
it('sends pane identity beside incarnation for relays that only understand the legacy fence', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
|
||||
const attachForReconnect = vi.fn().mockResolvedValue({ incarnationId: 'incarnation-lease' })
|
||||
vi.mocked(getSshPtyProvider).mockReturnValue({
|
||||
attachForReconnect,
|
||||
dispose: vi.fn()
|
||||
} as unknown as ReturnType<typeof getSshPtyProvider>)
|
||||
vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([
|
||||
{ ...detachedLease(), incarnationId: 'incarnation-lease' }
|
||||
] as ReturnType<typeof mockStore.getSshRemotePtyLeases>)
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
|
||||
await session.establish(mockConn)
|
||||
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-live', undefined, 'incarnation-lease', {
|
||||
paneKey: `tab-1:${INCARNATION_LEAF_ID}`,
|
||||
tabId: 'tab-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('does not restore a PTY whose matching exit shares the attach reply batch', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps()
|
||||
const incarnationId = 'incarnation-exited-during-attach'
|
||||
@@ -518,8 +489,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => {
|
||||
expect(runtime.registerPty).not.toHaveBeenCalled()
|
||||
expect(restorePtyIncarnation).toHaveBeenCalledWith(APP_PTY_ID, incarnationId)
|
||||
expect(setPtyOwnership).not.toHaveBeenCalled()
|
||||
// Step F: observed at the one bind producer, else this negative is vacuous.
|
||||
expect(bindPaneShell).not.toHaveBeenCalled()
|
||||
expect(mockStore.persistPtyBinding).not.toHaveBeenCalled()
|
||||
expect(sourceActivationLease.rollback).toHaveBeenCalledOnce()
|
||||
expect(sourceActivationLease.commit).not.toHaveBeenCalled()
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
@@ -578,14 +548,8 @@ describe('SshRelaySession reconnect incarnation ordering', () => {
|
||||
incarnationId: currentIncarnationId
|
||||
})
|
||||
expect(setPtyOwnership).toHaveBeenCalledWith(APP_PTY_ID, 'target-1')
|
||||
// Step F moved the observation point to the one bind producer; the asserted
|
||||
// intent — exact incarnation proof, mayCreate:false — is unchanged.
|
||||
expect(bindPaneShell).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
ptyId: APP_PTY_ID,
|
||||
incarnationId: currentIncarnationId,
|
||||
mayCreate: false
|
||||
})
|
||||
expect(mockStore.persistPtyBinding).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ ptyId: APP_PTY_ID, incarnationId: currentIncarnationId })
|
||||
)
|
||||
expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:replay', {
|
||||
id: APP_PTY_ID,
|
||||
@@ -603,9 +567,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => {
|
||||
vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([detachedLease()] as ReturnType<
|
||||
typeof mockStore.getSshRemotePtyLeases
|
||||
>)
|
||||
// Step F: the durable write now happens inside the bind producer, so that is
|
||||
// where the failure has to be injected.
|
||||
vi.mocked(bindPaneShell).mockImplementationOnce(() => {
|
||||
vi.mocked(mockStore.persistPtyBinding).mockImplementationOnce(() => {
|
||||
throw new Error('disk full')
|
||||
})
|
||||
const runtime = { onPtySpawned: vi.fn(), registerPty: vi.fn() }
|
||||
|
||||
@@ -705,7 +705,7 @@ describe('SshRelaySession recovery race fencing', () => {
|
||||
await session.reconnect(deps.mockConn)
|
||||
|
||||
expect(attachForReconnectMock).toHaveBeenCalledTimes(2)
|
||||
expect(attachForReconnectMock.mock.calls.at(-1)?.[1]).toMatchObject({
|
||||
expect(attachForReconnectMock.mock.calls.at(-1)?.[2]).toMatchObject({
|
||||
status: 'checkpoint',
|
||||
deliveryToken: 'new-token',
|
||||
acceptedSourceEndSu: 8
|
||||
@@ -763,7 +763,7 @@ describe('SshRelaySession recovery race fencing', () => {
|
||||
const replacementReconnect = session.reconnect(deps.mockConn)
|
||||
await Promise.all([staleReconnect, replacementReconnect])
|
||||
|
||||
const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[1])
|
||||
const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[2])
|
||||
expect(recoveryRequests).toHaveLength(2)
|
||||
expect(recoveryRequests[1]).toMatchObject({
|
||||
status: 'checkpoint',
|
||||
|
||||
@@ -54,7 +54,6 @@ type RejectedDeliverySession = {
|
||||
>
|
||||
rejectedPtyRecoveryAttempts: Map<string, unknown>
|
||||
sourceIdentityByRelayPtyId: Map<string, unknown>
|
||||
watchMuxForRelayLoss: (mux: SshChannelMultiplexer) => void
|
||||
retireExitedPty: (payload: {
|
||||
id: string
|
||||
code: number
|
||||
@@ -90,18 +89,9 @@ function rejectedPayload(overrides: Partial<SshPtyDataPayload> = {}): SshPtyData
|
||||
|
||||
function prepareSession() {
|
||||
const deps = createMockDeps()
|
||||
const disposeHandlers: ((reason: 'shutdown' | 'connection_lost') => void)[] = []
|
||||
const mux = {
|
||||
isDisposed: vi.fn(() => false),
|
||||
onDispose: vi.fn((handler: (reason: 'shutdown' | 'connection_lost') => void) => {
|
||||
disposeHandlers.push(handler)
|
||||
return () => {}
|
||||
}),
|
||||
dispose: vi.fn((reason: 'shutdown' | 'connection_lost' = 'shutdown') => {
|
||||
for (const handler of disposeHandlers) {
|
||||
handler(reason)
|
||||
}
|
||||
})
|
||||
dispose: vi.fn()
|
||||
} as unknown as SshChannelMultiplexer
|
||||
const session = new SshRelaySession(
|
||||
'target-1',
|
||||
@@ -111,11 +101,6 @@ function prepareSession() {
|
||||
)
|
||||
const internals = session as unknown as RejectedDeliverySession
|
||||
internals.mux = mux
|
||||
// Why wired: disposing this mux is what fires the host-wide relay-lost reconnect, so a test can
|
||||
// observe the blast radius of a per-PTY escalation instead of only the local dispose call.
|
||||
const relayLost = vi.fn()
|
||||
session.setOnRelayLost(relayLost)
|
||||
internals.watchMuxForRelayLoss(mux)
|
||||
internals.activePtyProviderGeneration = 23
|
||||
internals.ptyConsumerSessionState = {
|
||||
mode: 'negotiated',
|
||||
@@ -125,7 +110,7 @@ function prepareSession() {
|
||||
ownerLease: 'owner-lease',
|
||||
outputFlowControl: { version: 1, windowSu: 64 }
|
||||
}
|
||||
return { deps, internals, mux, relayLost, session }
|
||||
return { deps, internals, mux, session }
|
||||
}
|
||||
|
||||
describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
@@ -246,8 +231,7 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
const recovery = reattachKnownPty.mock.calls[0]?.[0]
|
||||
expect(recovery?.ptyId).toBe('pty-bad')
|
||||
expect(Array.from(recovery?.activeLeaseByPtyId.keys() ?? [])).toEqual(['pty-bad'])
|
||||
// The sibling `expectedIdentityByPtyId` clause is gone with the map itself (STA-3077 step A):
|
||||
// reattach no longer carries pane identity at all, which subsumes "carries none for this pty".
|
||||
expect(Array.from(recovery?.expectedIdentityByPtyId.keys() ?? [])).toEqual([])
|
||||
expect(recovery?.targetedDeliveryRecovery).toBe('fresh-activation')
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [
|
||||
'pty-bad'
|
||||
@@ -259,10 +243,8 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Why this one still drops the channel: the relay refused to confirm the delivery was canceled, so
|
||||
// no per-PTY fence is available — it is an unprovable state, not an exhausted retry count.
|
||||
it('reconnects instead of canceling an unprovable malformed delivery', async () => {
|
||||
const { internals, mux, relayLost } = prepareSession()
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
@@ -277,7 +259,6 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
)
|
||||
|
||||
expect(mux.dispose).toHaveBeenCalledWith('connection_lost')
|
||||
expect(relayLost).toHaveBeenCalledWith('target-1')
|
||||
expect(reattach).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -357,8 +338,7 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
internals.reattachRejectedPty('pty-bad', mux, 23, 'confirm-existing')
|
||||
).resolves.toBe(true)
|
||||
|
||||
// Two args since step A dropped the expected-identity parameter from the attach call.
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', checkpoint)
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', undefined, checkpoint)
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -377,11 +357,11 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
expect(internals.rejectedPtyRecoveryAttempts).toHaveLength(0)
|
||||
})
|
||||
|
||||
// Why this is the oracle: an exhausted retry budget is not proof of anything, and the relay channel
|
||||
// is shared — dropping it rotates provider authority, aborts every in-flight fs and git request on
|
||||
// the target and stalls every sibling PTY over one PTY's undeliverable output.
|
||||
it('parks one PTY after its recovery budget runs out and leaves the shared channel and a sibling alone', async () => {
|
||||
const { internals, mux, relayLost, session } = prepareSession()
|
||||
// Why a channel drop rather than a terminal relay error: a terminal error clears the reconnect
|
||||
// backoff and rotates provider authority, aborting every fs and git request on the target, so one
|
||||
// PTY's undeliverable output would strand the whole connection in manual recovery.
|
||||
it('bounds failed targeted recovery and escalates to a recoverable relay reconnect', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
@@ -390,90 +370,19 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('PTY pty-bad delivery recovery exhausted')
|
||||
),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
// Why the extra wait: a dispose would be scheduled off the same recovery pass as the warning.
|
||||
await new Promise((resolve) => setTimeout(resolve, 400))
|
||||
await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 })
|
||||
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
expect(relayLost).not.toHaveBeenCalled()
|
||||
expect(reattach).toHaveBeenCalledTimes(2)
|
||||
expect(reattach.mock.calls).toEqual([
|
||||
['pty-bad', mux, 23, 'confirm-existing'],
|
||||
['pty-bad', mux, 23, 'confirm-existing']
|
||||
])
|
||||
expect(mux.dispose).toHaveBeenCalledWith('connection_lost')
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('PTY pty-bad delivery recovery exhausted')
|
||||
)
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(reattach).toHaveBeenCalledTimes(12)
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: 'ssh:target-1@@pty-healthy',
|
||||
data: 'healthy',
|
||||
ptyIncarnation: 'incarnation-healthy',
|
||||
source: source({
|
||||
relayPtyId: 'pty-healthy',
|
||||
spanId: 'token-healthy:0:7',
|
||||
deliveryToken: 'token-healthy',
|
||||
sourceEndSu: 7
|
||||
}),
|
||||
sourceRejected: undefined
|
||||
})
|
||||
)
|
||||
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledOnce()
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'ssh:target-1@@pty-healthy', data: 'healthy' })
|
||||
)
|
||||
})
|
||||
|
||||
// The clause above pins the containment; this one pins that containment is not abandonment. The
|
||||
// park's only stated escape is the next relay open, and a shared channel that stays healthy never
|
||||
// gives it one — so a pane could sit with no output and no way back, which is the exact state
|
||||
// this change exists to prevent. The park has to expire.
|
||||
it('re-arms a parked PTY once its cooldown has passed rather than leaving it dark', async () => {
|
||||
const { internals } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('PTY pty-bad delivery recovery exhausted')
|
||||
),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
expect(reattach).toHaveBeenCalledTimes(12)
|
||||
|
||||
// Let the exhaustion loop go quiet first: its last retries are timer-driven and would
|
||||
// otherwise land after the clear below and satisfy the final clause on their own.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500))
|
||||
reattach.mockClear()
|
||||
|
||||
// Every frame below reuses the ORIGINAL delivery token, because that is what actually happens:
|
||||
// retirement is ours, not the host's, and a stalled source keeps publishing the token we
|
||||
// retired. It also matters that they are identical — retirement holds ONE key per relay PTY,
|
||||
// so slipping a different token in here would overwrite that key, un-retire the original and
|
||||
// let the final clause pass with the wake-up path still broken.
|
||||
|
||||
// Before the cooldown: parked, so nothing retries.
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await new Promise((resolve) => setTimeout(resolve, 600))
|
||||
expect(reattach).not.toHaveBeenCalled()
|
||||
|
||||
// Backdate the park instead of burning a minute of real time — the property under test is
|
||||
// elapsed time, not which timer implementation measures it.
|
||||
const parked = internals.rejectedPtyRecoveryAttempts.get('ssh:target-1@@pty-bad') as {
|
||||
parkedAt: number
|
||||
}
|
||||
parked.parkedAt -= 60_000
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
|
||||
await vi.waitFor(() => expect(reattach).toHaveBeenCalled(), { timeout: 10_000 })
|
||||
})
|
||||
|
||||
it('stops without an error when the rejected PTY exited during recovery', async () => {
|
||||
@@ -495,30 +404,6 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// The counterpart to the clause above, and the distinction the whole design rests on. hasPty is
|
||||
// three-state: null means the provider has not listed the host yet, which is precisely the state
|
||||
// a fresh provider is in right after the reconnect that produces rejected frames. Reading that
|
||||
// as death drops the attempt and schedules nothing, and the delivery token is already retired,
|
||||
// so nothing can revive it — the pane stays dark. Only an explicit false is proof.
|
||||
it('keeps retrying while the PTY liveness is merely unknown', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => null } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
|
||||
// More than one attempt is the whole point: unknown liveness must not end recovery.
|
||||
await vi.waitFor(() => expect(reattach.mock.calls.length).toBeGreaterThan(1), {
|
||||
timeout: 10_000
|
||||
})
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Why bounded: the bulk reconnect path caps the identical attach work at 8, and a relay that
|
||||
// starts rejecting across every PTY at once would otherwise open one attach round trip per PTY.
|
||||
it('caps concurrent targeted reattaches and coalesces repeats for one PTY', async () => {
|
||||
@@ -567,18 +452,18 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
})
|
||||
|
||||
it('does not let accepted frames refill the recovery budget indefinitely', async () => {
|
||||
const { internals, mux, relayLost, session } = prepareSession()
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
// Why alternating, with a fresh bad token each round: every rejection retires its own delivery,
|
||||
// so a flapping PTY only keeps asking for recovery by moving onto new ones, and the accepted
|
||||
// frame in between is what used to clear the budget outright — only the per-generation ceiling
|
||||
// can stop it.
|
||||
// frame in between is what used to clear the budget outright. Each reattach here succeeds, so
|
||||
// the consecutive budget is cleared legitimately too — only the per-generation ceiling can stop
|
||||
// it.
|
||||
for (let round = 0; round < 40; round++) {
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
@@ -602,17 +487,9 @@ describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
)
|
||||
await Promise.resolve()
|
||||
}
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('PTY pty-bad delivery recovery exhausted')
|
||||
),
|
||||
{ timeout: 2000 }
|
||||
)
|
||||
await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 })
|
||||
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
expect(relayLost).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledTimes(40)
|
||||
expect(reattach.mock.calls.length).toBeLessThanOrEqual(12)
|
||||
})
|
||||
|
||||
@@ -104,7 +104,6 @@ function createMockDeps(): {
|
||||
upsertSshPtyConsumerRecovery: vi.fn(),
|
||||
removeSshPtyConsumerRecovery: vi.fn(),
|
||||
getSshRemotePtyLeases: vi.fn().mockReturnValue([]),
|
||||
supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0),
|
||||
markSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLeases: vi.fn(),
|
||||
markSshRemotePtyLeasesAsync: vi.fn(),
|
||||
|
||||
@@ -21,14 +21,12 @@ export function createMockDeps(): SshRelaySessionTestDeps {
|
||||
upsertSshPtyConsumerRecovery: vi.fn(),
|
||||
removeSshPtyConsumerRecovery: vi.fn(),
|
||||
getSshRemotePtyLeases: vi.fn().mockReturnValue([]),
|
||||
supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0),
|
||||
markSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLeases: vi.fn(),
|
||||
markSshRemotePtyLeasesAsync: vi.fn(),
|
||||
markSshRemotePtyLeasesForShutdown: vi.fn(),
|
||||
markSshRemotePtyLeasesAttachedAsync: vi.fn(),
|
||||
markSshRemotePtyLeasesTerminatedAsync: vi.fn(),
|
||||
persistPtyBinding: vi.fn().mockReturnValue(true)
|
||||
persistPtyBinding: vi.fn()
|
||||
} as unknown as Store
|
||||
const mockPortForward = {
|
||||
removeAllForwards: vi.fn()
|
||||
|
||||
@@ -1,402 +0,0 @@
|
||||
/**
|
||||
* Journey 4 — two distinct final-host connections live at the same time, and one
|
||||
* host's delivery failure may not reach the other host or a sibling PTY.
|
||||
*
|
||||
* Everything below SSH is real: two `SshRelaySession`s, two real
|
||||
* `SshChannelMultiplexer`s, two real `SshPtyProvider`s, and two real
|
||||
* `RelayDispatcher`s speaking the relay wire over an in-process pipe. The fault
|
||||
* is injected the way a desynced relay actually produces it — a `pty.data`
|
||||
* frame whose source header does not parse, with a delivery identity the host
|
||||
* cannot cancel — so the whole rejection/recovery machinery runs for real.
|
||||
*
|
||||
* The oracle is the blast radius of exhausting ONE PTY's delivery-recovery
|
||||
* budget: the shared relay channel, the sibling PTY on the same host, and the
|
||||
* entire second host must be untouched.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import type { Store } from '../persistence'
|
||||
import type { SshPortForwardManager } from './ssh-port-forward'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import type { MultiplexerTransport } from './ssh-channel-multiplexer'
|
||||
import { RelayDispatcher } from '../../relay/dispatcher'
|
||||
import { getSshPtyProvider, unregisterSshPtyProvider } from '../ipc/pty'
|
||||
import type { SshPtyProvider } from '../providers/ssh-pty-provider'
|
||||
import { toAppSshPtyId } from '../providers/ssh-pty-id'
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
installSshPtyOutputIntake,
|
||||
type acceptSshPtyOutputData
|
||||
} from '../ipc/ssh-pty-output-intake-registry'
|
||||
import type { SshPtyOutputIntake } from '../ipc/ssh-pty-output-intake'
|
||||
|
||||
vi.mock('./ssh-relay-deploy', () => ({
|
||||
deployAndLaunchRelay: vi.fn()
|
||||
}))
|
||||
|
||||
const { deployAndLaunchRelay } = await import('./ssh-relay-deploy')
|
||||
const { SshRelaySession } = await import('./ssh-relay-session')
|
||||
|
||||
type DeliveredFrame = Parameters<typeof acceptSshPtyOutputData>[0]
|
||||
|
||||
/** The relay-side half of one host: a real dispatcher plus wire-level controls. */
|
||||
type FakeRelayHost = {
|
||||
transport: MultiplexerTransport
|
||||
dispatcher: RelayDispatcher
|
||||
/** Publishes output on the PTY's live delivery, advancing its source cursor. */
|
||||
publish: (relayPtyId: string, data: string) => void
|
||||
/**
|
||||
* Publishes a frame whose source header does not parse (`sourceLengthSu`
|
||||
* disagrees with the payload) but which still carries a complete delivery
|
||||
* identity this client never installed. That combination is what routes the
|
||||
* rejection into the bounded per-PTY recovery budget rather than an immediate
|
||||
* channel reconnect.
|
||||
*/
|
||||
publishMalformed: (relayPtyId: string, data: string) => void
|
||||
requestCounts: Map<string, number>
|
||||
/** When true, `pty.attach` is refused — the relay cannot re-prove the delivery. */
|
||||
refuseAttach: { value: boolean }
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
function createFakeRelayHost(): FakeRelayHost {
|
||||
const clientDataCallbacks: ((data: Buffer) => void)[] = []
|
||||
const clientCloseCallbacks: (() => void)[] = []
|
||||
const requestCounts = new Map<string, number>()
|
||||
const refuseAttach = { value: false }
|
||||
const deliveries = new Map<
|
||||
string,
|
||||
{ deliveryToken: string; ptyIncarnation: string; sentSu: number }
|
||||
>()
|
||||
let spawnCount = 0
|
||||
|
||||
const transport: MultiplexerTransport = {
|
||||
write: (data) => {
|
||||
setImmediate(() => dispatcher.feed(data))
|
||||
},
|
||||
onData: (cb) => {
|
||||
clientDataCallbacks.push(cb)
|
||||
},
|
||||
onClose: (cb) => {
|
||||
clientCloseCallbacks.push(cb)
|
||||
},
|
||||
close: () => {
|
||||
for (const cb of clientCloseCallbacks) {
|
||||
cb()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dispatcher = new RelayDispatcher((data) => {
|
||||
setImmediate(() => {
|
||||
for (const cb of clientDataCallbacks) {
|
||||
cb(data)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
const count = (method: string): void => {
|
||||
requestCounts.set(method, (requestCounts.get(method) ?? 0) + 1)
|
||||
}
|
||||
|
||||
dispatcher.onRequest('pty.openClient', async (params) => {
|
||||
count('pty.openClient')
|
||||
return {
|
||||
protocolVersion: 1,
|
||||
serverBuildId: 'test-relay-build',
|
||||
clientGeneration: 1,
|
||||
role: 'session-owner',
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'test-owner-lease',
|
||||
resumed: params.resume !== undefined,
|
||||
capabilities: {
|
||||
outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
|
||||
}
|
||||
}
|
||||
})
|
||||
dispatcher.onRequest('session.resolveHome', async (params) => ({
|
||||
resolvedPath: params.path === '~' ? '/home/orca' : params.path
|
||||
}))
|
||||
dispatcher.onRequest('git.listWorktrees', async () => [])
|
||||
dispatcher.onRequest('ports.detect', async () => ({ ports: [], platform: 'linux' }))
|
||||
dispatcher.onRequest('pty.spawn', async () => {
|
||||
count('pty.spawn')
|
||||
spawnCount += 1
|
||||
const id = `remote-pty-${spawnCount}`
|
||||
const ptyIncarnation = `incarnation-${id}`
|
||||
const deliveryToken = `delivery-${id}`
|
||||
deliveries.set(id, { deliveryToken, ptyIncarnation, sentSu: 0 })
|
||||
return {
|
||||
id,
|
||||
incarnationId: ptyIncarnation,
|
||||
sourceActivation: {
|
||||
status: 'pending',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation,
|
||||
deliveryToken,
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
}
|
||||
}
|
||||
})
|
||||
dispatcher.onRequest('pty.getCwd', async () => {
|
||||
count('pty.getCwd')
|
||||
return '/home/orca/project'
|
||||
})
|
||||
// Why `canceled: false`: the host cannot retire a delivery identity it never
|
||||
// issued. That is the branch that asks the client to confirm the existing
|
||||
// delivery rather than to drop the channel, so recovery enters the bounded
|
||||
// retry budget this test is about.
|
||||
dispatcher.onRequest('pty.cancelDelivery', async () => {
|
||||
count('pty.cancelDelivery')
|
||||
return { canceled: false }
|
||||
})
|
||||
dispatcher.onRequest('pty.attach', async () => {
|
||||
count('pty.attach')
|
||||
if (refuseAttach.value) {
|
||||
// Why not a not-found error: not-found is positive proof the PTY is gone
|
||||
// and takes the destructive path. A refusal is the "unknown" case, which
|
||||
// is what keeps the recovery budget burning down.
|
||||
throw new Error('relay_attach_refused')
|
||||
}
|
||||
return { id: 'unused' }
|
||||
})
|
||||
|
||||
const delivery = (relayPtyId: string) => {
|
||||
const state = deliveries.get(relayPtyId)
|
||||
if (!state) {
|
||||
throw new Error(`relay has no delivery for ${relayPtyId}`)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
return {
|
||||
transport,
|
||||
dispatcher,
|
||||
publish: (relayPtyId, data) => {
|
||||
const state = delivery(relayPtyId)
|
||||
state.sentSu += data.length
|
||||
dispatcher.notify('pty.data', {
|
||||
id: relayPtyId,
|
||||
data,
|
||||
deliveryToken: state.deliveryToken,
|
||||
ptyIncarnation: state.ptyIncarnation,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
sourceEndSu: state.sentSu,
|
||||
sourceLengthSu: data.length
|
||||
})
|
||||
},
|
||||
publishMalformed: (relayPtyId, data) => {
|
||||
dispatcher.notify('pty.data', {
|
||||
id: relayPtyId,
|
||||
data,
|
||||
deliveryToken: 'delivery-this-client-never-installed',
|
||||
ptyIncarnation: 'incarnation-unknown',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
sourceEndSu: data.length + 1,
|
||||
sourceLengthSu: data.length + 1
|
||||
})
|
||||
},
|
||||
requestCounts,
|
||||
refuseAttach,
|
||||
dispose: () => dispatcher.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function createStore(): Store {
|
||||
return {
|
||||
getRepos: vi.fn().mockReturnValue([]),
|
||||
getSshPtyConsumerRecovery: vi.fn().mockReturnValue(null),
|
||||
upsertSshPtyConsumerRecovery: vi.fn(),
|
||||
removeSshPtyConsumerRecovery: vi.fn(),
|
||||
getSshRemotePtyLeases: vi.fn().mockReturnValue([]),
|
||||
supersedeDuplicatePaneLeases: vi.fn().mockReturnValue(0),
|
||||
markSshRemotePtyLease: vi.fn(),
|
||||
markSshRemotePtyLeases: vi.fn(),
|
||||
markSshRemotePtyLeasesAsync: vi.fn(),
|
||||
markSshRemotePtyLeasesForShutdown: vi.fn(),
|
||||
markSshRemotePtyLeasesAttachedAsync: vi.fn(),
|
||||
persistPtyBinding: vi.fn().mockReturnValue(true)
|
||||
} as unknown as Store
|
||||
}
|
||||
|
||||
type LiveHost = {
|
||||
targetId: string
|
||||
relay: FakeRelayHost
|
||||
session: InstanceType<typeof SshRelaySession>
|
||||
provider: SshPtyProvider
|
||||
relayLost: ReturnType<typeof vi.fn>
|
||||
terminalError: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
||||
async function openHost(targetId: string): Promise<LiveHost> {
|
||||
const relay = createFakeRelayHost()
|
||||
vi.mocked(deployAndLaunchRelay).mockResolvedValue({
|
||||
transport: relay.transport,
|
||||
serverBuildId: 'test-relay-build',
|
||||
platform: 'linux-x64'
|
||||
})
|
||||
const session = new SshRelaySession(
|
||||
targetId,
|
||||
vi.fn().mockReturnValue({
|
||||
isDestroyed: () => false,
|
||||
isVisible: () => true,
|
||||
isMinimized: () => false,
|
||||
webContents: { send: vi.fn() }
|
||||
}),
|
||||
createStore(),
|
||||
{ removeAllForwards: vi.fn().mockResolvedValue(undefined) } as unknown as SshPortForwardManager
|
||||
)
|
||||
const relayLost = vi.fn()
|
||||
const terminalError = vi.fn()
|
||||
session.setOnRelayLost(relayLost)
|
||||
session.setOnTerminalRelayError(terminalError)
|
||||
await session.establish({} as SshConnection)
|
||||
const provider = getSshPtyProvider(targetId) as SshPtyProvider | undefined
|
||||
if (!provider) {
|
||||
throw new Error(`no PTY provider registered for ${targetId}`)
|
||||
}
|
||||
return { targetId, relay, session, provider, relayLost, terminalError }
|
||||
}
|
||||
|
||||
describe('two live SSH hosts over real relay dispatchers', () => {
|
||||
const hosts: LiveHost[] = []
|
||||
let delivered: DeliveredFrame[] = []
|
||||
let uninstallIntake: (() => void) | null = null
|
||||
let warnSpy: ReturnType<typeof vi.spyOn>
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
delivered = []
|
||||
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
// Why the real registration seam: the session publishes accepted output
|
||||
// through `acceptSshPtyOutputData`, so installing a recording intake
|
||||
// observes the production delivery boundary without stubbing the module.
|
||||
uninstallIntake = installSshPtyOutputIntake({
|
||||
acceptData: async (event) => {
|
||||
delivered.push(event)
|
||||
return { status: 'accepted' } as never
|
||||
},
|
||||
acceptExit: async () => {},
|
||||
closeGeneration: () => {},
|
||||
dispose: () => {}
|
||||
} as unknown as SshPtyOutputIntake)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
for (const host of hosts.splice(0)) {
|
||||
host.session.dispose()
|
||||
host.relay.dispose()
|
||||
unregisterSshPtyProvider(host.targetId)
|
||||
}
|
||||
uninstallIntake?.()
|
||||
uninstallIntake = null
|
||||
warnSpy.mockRestore()
|
||||
})
|
||||
|
||||
/**
|
||||
* Brings both hosts up, gives host A two PTYs and host B one, proves all three
|
||||
* deliver, then burns host A's PTY-1 delivery-recovery budget to exhaustion.
|
||||
*
|
||||
* The two clauses are separate tests on purpose: a mutation that only breaks
|
||||
* same-host containment must be seen to leave the cross-host clause green,
|
||||
* which proves the clauses independently rather than jointly.
|
||||
*/
|
||||
async function exhaustOnePtyOnHostA(): Promise<{
|
||||
hostA: LiveHost
|
||||
hostB: LiveHost
|
||||
a2: { id: string }
|
||||
b1: { id: string }
|
||||
}> {
|
||||
const hostA = await openHost('host-a')
|
||||
const hostB = await openHost('host-b')
|
||||
hosts.push(hostA, hostB)
|
||||
|
||||
const a1 = await hostA.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' })
|
||||
const a2 = await hostA.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' })
|
||||
const b1 = await hostB.provider.spawn({ cols: 80, rows: 24, cwd: '/home/orca' })
|
||||
expect(a1.id).toBe(toAppSshPtyId('host-a', 'remote-pty-1'))
|
||||
expect(a2.id).toBe(toAppSshPtyId('host-a', 'remote-pty-2'))
|
||||
expect(b1.id).toBe(toAppSshPtyId('host-b', 'remote-pty-1'))
|
||||
// Two distinct final-host connections, each with its own provider identity.
|
||||
expect(hostA.provider).not.toBe(hostB.provider)
|
||||
|
||||
hostA.relay.publish('remote-pty-1', 'a1-before')
|
||||
hostA.relay.publish('remote-pty-2', 'a2-before')
|
||||
hostB.relay.publish('remote-pty-1', 'b1-before')
|
||||
await vi.waitFor(() => expect(delivered).toHaveLength(3), { timeout: 5_000 })
|
||||
delivered = []
|
||||
|
||||
// The fault: one PTY on host A can never re-prove its delivery.
|
||||
hostA.relay.refuseAttach.value = true
|
||||
hostA.relay.publishMalformed('remote-pty-1', 'a1-poisoned')
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('remote-pty-1 delivery recovery exhausted')
|
||||
),
|
||||
{ timeout: 60_000 }
|
||||
)
|
||||
// Why the extra settle: a channel drop is scheduled off the same recovery
|
||||
// pass that logs the exhaustion, not before it.
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
|
||||
// The budget was really spent on relay round trips, so the fault is live.
|
||||
expect(hostA.relay.requestCounts.get('pty.attach') ?? 0).toBeGreaterThanOrEqual(12)
|
||||
expect(hostA.relay.requestCounts.get('pty.cancelDelivery') ?? 0).toBeGreaterThan(0)
|
||||
// The undeliverable frame itself never reached a consumer.
|
||||
expect(delivered.some((frame) => frame.data === 'a1-poisoned')).toBe(false)
|
||||
|
||||
return { hostA, hostB, a2, b1 }
|
||||
}
|
||||
|
||||
it('leaves the sibling PTY and the shared channel on the same host working', async () => {
|
||||
const { hostA, a2 } = await exhaustOnePtyOnHostA()
|
||||
|
||||
hostA.relay.publish('remote-pty-2', 'a2-after')
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(
|
||||
delivered.filter((frame) => frame.id === a2.id && frame.data === 'a2-after'),
|
||||
'the sibling PTY stopped receiving output after its neighbour was parked'
|
||||
).toHaveLength(1),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
// A live request round trip: the shared channel still carries the fs/git
|
||||
// control plane that dropping it would have aborted.
|
||||
await expect(hostA.provider.getCwd(a2.id)).resolves.toBe('/home/orca/project')
|
||||
expect(hostA.relayLost, 'host A lost its relay channel over one PTY').not.toHaveBeenCalled()
|
||||
expect(hostA.terminalError, 'host A was parked in manual recovery').not.toHaveBeenCalled()
|
||||
}, 120_000)
|
||||
|
||||
it('leaves the second host entirely untouched', async () => {
|
||||
const { hostB, b1 } = await exhaustOnePtyOnHostA()
|
||||
|
||||
expect(hostB.relayLost, 'host B lost its relay channel').not.toHaveBeenCalled()
|
||||
expect(hostB.terminalError, 'host B was parked in manual recovery').not.toHaveBeenCalled()
|
||||
expect(
|
||||
hostB.relay.requestCounts.get('pty.attach') ?? 0,
|
||||
"host A's recovery reached host B's relay"
|
||||
).toBe(0)
|
||||
expect(
|
||||
hostB.relay.requestCounts.get('pty.cancelDelivery') ?? 0,
|
||||
"host A's delivery cancellation reached host B's relay"
|
||||
).toBe(0)
|
||||
expect(
|
||||
hostB.relay.requestCounts.get('pty.openClient') ?? 0,
|
||||
'host B renegotiated its consumer session'
|
||||
).toBe(1)
|
||||
await expect(hostB.provider.getCwd(b1.id)).resolves.toBe('/home/orca/project')
|
||||
hostB.relay.publish('remote-pty-1', 'b1-after')
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(
|
||||
delivered.filter((frame) => frame.id === b1.id && frame.data === 'b1-after')
|
||||
).toHaveLength(1),
|
||||
{ timeout: 10_000 }
|
||||
)
|
||||
}, 120_000)
|
||||
})
|
||||
@@ -531,8 +531,26 @@ describe('SshRelaySession', () => {
|
||||
)
|
||||
})
|
||||
|
||||
// New relays prioritize incarnation; old relays ignore that additive field and need this fence.
|
||||
it('preserves pane identity for relays that predate incarnation fencing', async () => {
|
||||
it('forwards a lease tab identity to reattach so a reset relay cannot cross-wire it', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
|
||||
const { getSshPtyProvider } = await import('../ipc/pty')
|
||||
const mockAttach = vi.fn().mockResolvedValue(undefined)
|
||||
vi.mocked(getSshPtyProvider).mockReturnValue({
|
||||
attachForReconnect: mockAttach,
|
||||
dispose: vi.fn()
|
||||
} as unknown as ReturnType<typeof getSshPtyProvider>)
|
||||
vi.mocked(getPtyIdsForConnection).mockReturnValue([])
|
||||
vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([
|
||||
{ targetId: 'target-1', ptyId: 'pty-1', state: 'detached', tabId: 'tab-a' }
|
||||
] as ReturnType<typeof mockStore.getSshRemotePtyLeases>)
|
||||
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
await session.establish(mockConn)
|
||||
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-1', { tabId: 'tab-a' })
|
||||
})
|
||||
|
||||
it('forwards a lease pane identity when leaf identity is available', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
|
||||
const { getSshPtyProvider } = await import('../ipc/pty')
|
||||
const mockAttach = vi.fn().mockResolvedValue(undefined)
|
||||
@@ -549,40 +567,7 @@ describe('SshRelaySession', () => {
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
await session.establish(mockConn)
|
||||
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-1', undefined, undefined, {
|
||||
paneKey: `tab-a:${leafId}`,
|
||||
tabId: 'tab-a'
|
||||
})
|
||||
})
|
||||
|
||||
// This is the path that reattaches every known pty when a relay comes back — precisely when ids
|
||||
// have been reissued from pty-1 — so it is the one that most needs to say which shell it means.
|
||||
// The shell's own identity is authoritative; pane identity remains the fallback for old relays.
|
||||
it('forwards the shell identity its lease recorded when reattaching', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
|
||||
const { getSshPtyProvider } = await import('../ipc/pty')
|
||||
const mockAttach = vi.fn().mockResolvedValue(undefined)
|
||||
vi.mocked(getSshPtyProvider).mockReturnValue({
|
||||
attachForReconnect: mockAttach,
|
||||
dispose: vi.fn()
|
||||
} as unknown as ReturnType<typeof getSshPtyProvider>)
|
||||
vi.mocked(getPtyIdsForConnection).mockReturnValue([])
|
||||
const leafId = '11111111-1111-4111-8111-111111111111'
|
||||
vi.mocked(mockStore.getSshRemotePtyLeases).mockReturnValue([
|
||||
{
|
||||
targetId: 'target-1',
|
||||
ptyId: 'pty-1',
|
||||
state: 'detached',
|
||||
tabId: 'tab-a',
|
||||
leafId,
|
||||
incarnationId: 'inc-host-1'
|
||||
}
|
||||
] as ReturnType<typeof mockStore.getSshRemotePtyLeases>)
|
||||
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
await session.establish(mockConn)
|
||||
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-1', undefined, 'inc-host-1', {
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-1', {
|
||||
paneKey: `tab-a:${leafId}`,
|
||||
tabId: 'tab-a'
|
||||
})
|
||||
@@ -617,10 +602,13 @@ describe('SshRelaySession', () => {
|
||||
|
||||
await session.reconnect(mockConn)
|
||||
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-1', {
|
||||
paneKey: `tab-old:${staleLeafId}`,
|
||||
tabId: 'tab-old'
|
||||
})
|
||||
expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-1')
|
||||
expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-1')
|
||||
expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith('target-1', 'pty-1', 'expired')
|
||||
expect(mockAttach).toHaveBeenCalledOnce()
|
||||
expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', {
|
||||
id: 'ssh:target-1@@pty-1',
|
||||
code: -1
|
||||
@@ -683,11 +671,7 @@ describe('SshRelaySession', () => {
|
||||
expect(mockStore.markSshRemotePtyLeasesAttachedAsync).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// INVERTED for STA-3077 step E-0. This case previously asserted that a PTY the relay reports as
|
||||
// not-found is invalidated and broadcast as an exit. A not-found is not proof of death, so the
|
||||
// broadcast was a fabricated one; what must still hold is that the healthy sibling reattaches and
|
||||
// the unproven one is left alone rather than written off.
|
||||
it('leaves an unreattachable remote PTY alone while its sibling reattaches', async () => {
|
||||
it('invalidates and broadcasts remote PTYs that cannot reattach after relay reconnect', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps()
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
await session.establish(mockConn)
|
||||
@@ -709,9 +693,12 @@ describe('SshRelaySession', () => {
|
||||
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-stale')
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-live')
|
||||
expect(clearProviderPtyState).not.toHaveBeenCalledWith('ssh:target-1@@pty-stale')
|
||||
expect(deletePtyOwnership).not.toHaveBeenCalledWith('ssh:target-1@@pty-stale')
|
||||
expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything())
|
||||
expect(clearProviderPtyState).toHaveBeenCalledWith('ssh:target-1@@pty-stale')
|
||||
expect(deletePtyOwnership).toHaveBeenCalledWith('ssh:target-1@@pty-stale')
|
||||
expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:exit', {
|
||||
id: 'ssh:target-1@@pty-stale',
|
||||
code: -1
|
||||
})
|
||||
})
|
||||
|
||||
it('retries transient reattach failure without tearing down provider registration', async () => {
|
||||
|
||||
+110
-188
@@ -12,7 +12,6 @@ import type { SshPtyAttachResult } from '../providers/ssh-pty-session-reattach'
|
||||
import type { SshPtyDataCallback, SshPtyExitCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import type { SshPtyRecoveryActivationLease } from '../providers/ssh-pty-notification-routing'
|
||||
import { isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors'
|
||||
import { parseMatchingPtyExitedError } from '../../shared/ssh-pty-failure-tokens'
|
||||
import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id'
|
||||
import { SshFilesystemProvider } from '../providers/ssh-filesystem-provider'
|
||||
import { isMethodNotFoundError } from './ssh-filesystem-stream-reader'
|
||||
@@ -33,8 +32,6 @@ import {
|
||||
import { _internals as openCodeInternals } from '../opencode/hook-service'
|
||||
import { getPiAgentStatusExtensionSource } from '../pi/agent-status-extension-source'
|
||||
import {
|
||||
bindPaneShell,
|
||||
resolvePaneShellTabId,
|
||||
registerSshPtyProvider,
|
||||
unregisterSshPtyProvider,
|
||||
getSshPtyProvider,
|
||||
@@ -98,8 +95,6 @@ import {
|
||||
parseRemoteOrcaCliPostOutput
|
||||
} from './ssh-remote-orchestration-post-output'
|
||||
import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host'
|
||||
import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id'
|
||||
import { isValidTerminalTabId } from '../../shared/terminal-tab-id'
|
||||
import {
|
||||
SSH_AI_VAULT_LIST_SESSIONS_METHOD,
|
||||
SSH_AI_VAULT_LIST_SESSIONS_TIMEOUT_MS,
|
||||
@@ -108,6 +103,8 @@ import {
|
||||
type SshAiVaultRelayListParams,
|
||||
type SshAiVaultRelayTitleParams
|
||||
} from '../../shared/ssh-ai-vault-relay'
|
||||
import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id'
|
||||
import { isValidTerminalTabId } from '../../shared/terminal-tab-id'
|
||||
import {
|
||||
openSshPtyConsumerSession,
|
||||
type OpenSshPtyConsumerSessionOptions,
|
||||
@@ -141,15 +138,11 @@ const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8
|
||||
const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000
|
||||
const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50
|
||||
const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200
|
||||
// Why a ceiling at all: a PTY that alternates recovered and rejected frames would otherwise reattach
|
||||
// forever — each one costs a store read, an attach round trip and a store write. Exhausting it parks
|
||||
// this PTY's delivery; it never authorizes tearing anything down.
|
||||
const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2
|
||||
// Why a second ceiling: the consecutive budget resets whenever a reattach succeeds, so a PTY that
|
||||
// alternates recovered and rejected frames would otherwise reattach forever — each one costs a
|
||||
// store read, an attach round trip and a store write.
|
||||
const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12
|
||||
// Why a park expires: exhaustion parks one PTY's delivery instead of dropping the shared channel,
|
||||
// and the escape it names — the next relay open — may never come while that channel stays healthy.
|
||||
// A pane with no output and no way back is the state this change exists to prevent, so the park is
|
||||
// a cooldown rather than a verdict: recovery is rate-limited, never abandoned.
|
||||
const SSH_REJECTED_PTY_RECOVERY_PARK_MS = 60_000
|
||||
const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150
|
||||
const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed'
|
||||
|
||||
@@ -198,15 +191,15 @@ type RemoteCliBridgeEnv = {
|
||||
pathDelimiter?: ':' | ';'
|
||||
}
|
||||
|
||||
type LegacyExpectedPtyIdentity = { paneKey?: string; tabId?: string }
|
||||
type ExpectedPtyIdentity = { paneKey?: string; tabId?: string }
|
||||
type TargetedDeliveryRecovery = 'confirm-existing' | 'fresh-activation'
|
||||
|
||||
function legacyExpectedIdentityForLease(lease: {
|
||||
function expectedIdentityForLease(lease: {
|
||||
tabId?: string
|
||||
leafId?: string
|
||||
}): LegacyExpectedPtyIdentity | undefined {
|
||||
}): ExpectedPtyIdentity | null {
|
||||
if (typeof lease.tabId !== 'string' || lease.tabId.length === 0) {
|
||||
return undefined
|
||||
return null
|
||||
}
|
||||
const paneKey =
|
||||
isValidTerminalTabId(lease.tabId) &&
|
||||
@@ -214,7 +207,10 @@ function legacyExpectedIdentityForLease(lease: {
|
||||
isTerminalLeafId(lease.leafId)
|
||||
? makePaneKey(lease.tabId, lease.leafId)
|
||||
: undefined
|
||||
return { ...(paneKey ? { paneKey } : {}), tabId: lease.tabId }
|
||||
return {
|
||||
...(paneKey ? { paneKey } : {}),
|
||||
tabId: lease.tabId
|
||||
}
|
||||
}
|
||||
|
||||
function parseRecoveryComplete(params: Record<string, unknown>): PtySourceRecoveryComplete | null {
|
||||
@@ -347,10 +343,9 @@ export class SshRelaySession {
|
||||
string,
|
||||
{
|
||||
providerGeneration: number
|
||||
attempts: number
|
||||
generationAttempts: number
|
||||
reported: boolean
|
||||
/** When the budget ran out; 0 while it has not. Drives the park cooldown. */
|
||||
parkedAt: number
|
||||
}
|
||||
>()
|
||||
private readonly rejectedPtyRecoveryRetries = new Set<ReturnType<typeof setTimeout>>()
|
||||
@@ -1794,8 +1789,7 @@ export class SshRelaySession {
|
||||
const offeredSource = payload.source
|
||||
if (
|
||||
offeredSource &&
|
||||
this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource) &&
|
||||
!this.isParkedRecoveryDue(payload.id)
|
||||
this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource)
|
||||
) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
@@ -1852,19 +1846,6 @@ export class SshRelaySession {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* A parked PTY whose cooldown has elapsed must be let back past the retired-delivery filter.
|
||||
* Retirement is ours, not the host's: a stalled source keeps publishing the very token we
|
||||
* retired, so that filter would swallow every frame that could re-arm the park and the pane
|
||||
* would stay dark for good — the failure the cooldown exists to rule out. Letting one through
|
||||
* costs nothing: it is re-classified as rejected and re-retired, so no output reaches the
|
||||
* terminal; it only regains the ability to ask for recovery.
|
||||
*/
|
||||
private isParkedRecoveryDue(appPtyId: string): boolean {
|
||||
const attempt = this.rejectedPtyRecoveryAttempts.get(appPtyId)
|
||||
return !!attempt?.parkedAt && Date.now() - attempt.parkedAt >= SSH_REJECTED_PTY_RECOVERY_PARK_MS
|
||||
}
|
||||
|
||||
private recoverRejectedPtyDelivery(
|
||||
payload: SshPtyDataPayload,
|
||||
source: SshPtyDataPayload['source']
|
||||
@@ -1899,28 +1880,26 @@ export class SshRelaySession {
|
||||
const attempt =
|
||||
previous?.providerGeneration === providerGeneration
|
||||
? previous
|
||||
: { providerGeneration, generationAttempts: 0, reported: false, parkedAt: 0 }
|
||||
if (attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS) {
|
||||
// Why park instead of dropping the relay channel: a retry count is not proof of anything, and
|
||||
// the channel is shared — dropping it rotates provider authority, aborts every in-flight fs
|
||||
// and git request on the target and stalls every sibling PTY over one PTY's delivery. The
|
||||
// remote shell keeps running and its lease stands.
|
||||
if (attempt.parkedAt && Date.now() - attempt.parkedAt >= SSH_REJECTED_PTY_RECOVERY_PARK_MS) {
|
||||
// The park has served its cooldown. Re-arm rather than leave the pane dark for good.
|
||||
attempt.generationAttempts = 0
|
||||
attempt.reported = false
|
||||
attempt.parkedAt = 0
|
||||
} else {
|
||||
if (!attempt.reported) {
|
||||
attempt.reported = true
|
||||
attempt.parkedAt = Date.now()
|
||||
console.warn(
|
||||
`[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; parking its delivery for ${SSH_REJECTED_PTY_RECOVERY_PARK_MS}ms or until the next relay open`
|
||||
)
|
||||
}
|
||||
return
|
||||
: { providerGeneration, attempts: 0, generationAttempts: 0, reported: false }
|
||||
if (
|
||||
attempt.attempts >= SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS ||
|
||||
attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS
|
||||
) {
|
||||
if (!attempt.reported) {
|
||||
attempt.reported = true
|
||||
console.warn(
|
||||
`[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; dropping the relay channel to reconnect`
|
||||
)
|
||||
// Why a channel drop and not a terminal relay error: a terminal error clears the reconnect
|
||||
// backoff, rotates provider authority (aborting every in-flight fs and git request on the
|
||||
// target) and parks the target in a manual-recovery state — over one PTY's delivery. Losing
|
||||
// the channel is the recoverable escalation, and it is what this path did before targeted
|
||||
// recovery existed.
|
||||
mux.dispose('connection_lost')
|
||||
}
|
||||
return
|
||||
}
|
||||
attempt.attempts++
|
||||
attempt.generationAttempts++
|
||||
this.rejectedPtyRecoveryAttempts.set(appPtyId, attempt)
|
||||
void this.rejectedPtyReattaches
|
||||
@@ -1936,9 +1915,13 @@ export class SshRelaySession {
|
||||
)
|
||||
.then(
|
||||
(recovered) => {
|
||||
if (!recovered) {
|
||||
this.retryRejectedPtyDelivery(payload, source, appPtyId)
|
||||
if (recovered) {
|
||||
// Why only a completed reattach clears this: an accepted frame proves nothing about the
|
||||
// delivery that was rejected, and resetting on one lets a flapping PTY reattach forever.
|
||||
attempt.attempts = 0
|
||||
return
|
||||
}
|
||||
this.retryRejectedPtyDelivery(payload, source, appPtyId)
|
||||
},
|
||||
(error: unknown) => {
|
||||
console.warn(`[ssh-relay-session] PTY ${relayPtyId} targeted delivery recovery failed`, {
|
||||
@@ -1952,23 +1935,14 @@ export class SshRelaySession {
|
||||
|
||||
// Why liveness is checked before retrying: reattachKnownPty resolves without claiming the lease
|
||||
// when the PTY exited mid-attach, which is indistinguishable from a failed reattach at the call
|
||||
// site, so an ordinary exit would otherwise burn the whole recovery budget on attach round trips.
|
||||
// site. Retrying that race twice would drop the relay channel over an ordinary PTY exit.
|
||||
private retryRejectedPtyDelivery(
|
||||
payload: SshPtyDataPayload,
|
||||
source: SshPtyDataPayload['source'],
|
||||
appPtyId: string
|
||||
): void {
|
||||
const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined
|
||||
// Why `=== false` and not `!`: hasPty is three-state, and null means the provider has not
|
||||
// listed the host yet — ignorance, not death, which is exactly the state a fresh provider is
|
||||
// in right after the reconnect that produces these rejected frames. Treating it as death
|
||||
// dropped the recovery attempt and scheduled nothing, and since the delivery token is already
|
||||
// retired, no later frame could revive it: the pane stayed dark. Only proof stops us.
|
||||
if (
|
||||
!ptyProvider ||
|
||||
typeof ptyProvider.hasPty !== 'function' ||
|
||||
ptyProvider.hasPty(appPtyId) === false
|
||||
) {
|
||||
if (!ptyProvider || typeof ptyProvider.hasPty !== 'function' || !ptyProvider.hasPty(appPtyId)) {
|
||||
this.rejectedPtyRecoveryAttempts.delete(appPtyId)
|
||||
return
|
||||
}
|
||||
@@ -2005,25 +1979,21 @@ export class SshRelaySession {
|
||||
const activeLeaseByPtyId = activeLease
|
||||
? new Map<string, SshPtyLease>([[relayPtyId, activeLease]])
|
||||
: new Map<string, SshPtyLease>()
|
||||
const expectedIdentity = activeLease ? expectedIdentityForLease(activeLease) : undefined
|
||||
const attachedLeaseIds = new Set<string>()
|
||||
const terminatedLeaseIds = new Set<string>()
|
||||
await this.reattachKnownPty({
|
||||
ptyProvider,
|
||||
ptyId: relayPtyId,
|
||||
activeLeaseByPtyId,
|
||||
expectedIdentityByPtyId: expectedIdentity
|
||||
? new Map([[relayPtyId, expectedIdentity]])
|
||||
: new Map(),
|
||||
attachedLeaseIds,
|
||||
terminatedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue,
|
||||
targetedDeliveryRecovery
|
||||
})
|
||||
if (terminatedLeaseIds.size > 0) {
|
||||
await this.store.markSshRemotePtyLeasesTerminatedAsync(
|
||||
this.targetId,
|
||||
Array.from(terminatedLeaseIds)
|
||||
)
|
||||
}
|
||||
if (attachedLeaseIds.size > 0 && shouldContinue()) {
|
||||
await this.store.markSshRemotePtyLeasesAttachedAsync(
|
||||
this.targetId,
|
||||
@@ -2275,21 +2245,21 @@ export class SshRelaySession {
|
||||
mux: SshChannelMultiplexer,
|
||||
shouldContinue: () => boolean
|
||||
): Promise<void> {
|
||||
// Why: installs predating pane-keyed supersession carry duplicate live leases;
|
||||
// fanning out over them is what grafted panes the user never opened (STA-3077).
|
||||
const retired = await this.store.supersedeDuplicatePaneLeases(this.targetId)
|
||||
if (retired > 0) {
|
||||
console.info(
|
||||
`[ssh-relay-session] Retired ${retired} duplicate pane lease(s) for ${this.targetId}; their remote shells are left running.`
|
||||
)
|
||||
}
|
||||
const activeLeases = this.store
|
||||
.getSshRemotePtyLeases(this.targetId)
|
||||
.filter((lease) => lease.state !== 'terminated' && lease.state !== 'expired')
|
||||
const activeLeaseByPtyId = new Map(activeLeases.map((lease) => [lease.ptyId, lease]))
|
||||
const leasedPtyIds = activeLeases.map((lease) => lease.ptyId)
|
||||
// Why: pass pane identity so the relay can reject cross-generation id collisions; tabId falls back for pre-leafId leases.
|
||||
const expectedIdentityByPtyId = new Map(
|
||||
activeLeases
|
||||
.map((lease): [string, ExpectedPtyIdentity] | null => {
|
||||
const expected = expectedIdentityForLease(lease)
|
||||
return expected ? [lease.ptyId, expected] : null
|
||||
})
|
||||
.filter((entry): entry is [string, ExpectedPtyIdentity] => entry !== null)
|
||||
)
|
||||
const attachedLeaseIds = new Set<string>()
|
||||
const terminatedLeaseIds = new Set<string>()
|
||||
// Why: after app restart ptyOwnership is empty, but durable SSH leases still describe grace-window survivors.
|
||||
const ptyIds = Array.from(
|
||||
new Set([
|
||||
@@ -2316,8 +2286,8 @@ export class SshRelaySession {
|
||||
ptyProvider,
|
||||
ptyId,
|
||||
activeLeaseByPtyId,
|
||||
expectedIdentityByPtyId,
|
||||
attachedLeaseIds,
|
||||
terminatedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue
|
||||
@@ -2337,12 +2307,6 @@ export class SshRelaySession {
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(SSH_PTY_REATTACH_MAX_CONCURRENCY, ptyIds.length) }, worker)
|
||||
)
|
||||
if (terminatedLeaseIds.size > 0) {
|
||||
await this.store.markSshRemotePtyLeasesTerminatedAsync(
|
||||
this.targetId,
|
||||
Array.from(terminatedLeaseIds)
|
||||
)
|
||||
}
|
||||
if (attachedLeaseIds.size > 0 && shouldContinue()) {
|
||||
await this.store.markSshRemotePtyLeasesAttachedAsync(
|
||||
this.targetId,
|
||||
@@ -2355,8 +2319,8 @@ export class SshRelaySession {
|
||||
ptyProvider: SshPtyProvider
|
||||
ptyId: string
|
||||
activeLeaseByPtyId: Map<string, SshPtyLease>
|
||||
expectedIdentityByPtyId: Map<string, ExpectedPtyIdentity>
|
||||
attachedLeaseIds: Set<string>
|
||||
terminatedLeaseIds: Set<string>
|
||||
mux: SshChannelMultiplexer
|
||||
providerGeneration: number
|
||||
shouldContinue: () => boolean
|
||||
@@ -2366,8 +2330,8 @@ export class SshRelaySession {
|
||||
ptyProvider,
|
||||
ptyId,
|
||||
activeLeaseByPtyId,
|
||||
expectedIdentityByPtyId,
|
||||
attachedLeaseIds,
|
||||
terminatedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue,
|
||||
@@ -2394,17 +2358,12 @@ export class SshRelaySession {
|
||||
targetedDeliveryRecovery === 'fresh-activation'
|
||||
? undefined
|
||||
: await this.sourceRecoveryRequest(appPtyId)
|
||||
// The shell this lease names, as the host attested it. Without it the main reconnect path
|
||||
// attaches by id alone, and a replaced relay reissues ids from pty-1.
|
||||
const attachResult = await this.attachPtyWithRetry(
|
||||
ptyProvider,
|
||||
ptyId,
|
||||
expectedIdentityByPtyId.get(ptyId),
|
||||
recoveryRequest,
|
||||
shouldContinue,
|
||||
activeLeaseByPtyId.get(ptyId)?.incarnationId,
|
||||
activeLeaseByPtyId.get(ptyId)
|
||||
? legacyExpectedIdentityForLease(activeLeaseByPtyId.get(ptyId)!)
|
||||
: undefined
|
||||
shouldContinue
|
||||
)
|
||||
sourceActivationLease = attachResult.sourceActivationLease
|
||||
if (!shouldContinue()) {
|
||||
@@ -2550,13 +2509,7 @@ export class SshRelaySession {
|
||||
if (!shouldContinue()) {
|
||||
return
|
||||
}
|
||||
this.handlePtyReattachFailure(
|
||||
ptyId,
|
||||
pendingReattach,
|
||||
error,
|
||||
activeLeaseByPtyId.get(ptyId)?.incarnationId,
|
||||
terminatedLeaseIds
|
||||
)
|
||||
this.handlePtyReattachFailure(ptyId, appPtyId, pendingReattach, error)
|
||||
} finally {
|
||||
recoveryActivationLease?.retire()
|
||||
sourceActivationLease?.rollback()
|
||||
@@ -2595,42 +2548,22 @@ export class SshRelaySession {
|
||||
lease: SshPtyLease | undefined
|
||||
): void {
|
||||
if (lease?.worktreeId && lease.tabId && lease.leafId) {
|
||||
// Why bind before registering: `mayCreate: false` refuses when the pane is
|
||||
// gone, and registering first would surface a pane the user never opened.
|
||||
// A thrown write is unknown, not a refusal, so it must not detach anything.
|
||||
// The lease's tabId was frozen when it was written and the pane may have moved since, so the
|
||||
// live layout outranks it. Resolved before the write, so a throw cannot lose it and leave the
|
||||
// graph registered under the tab the pane left.
|
||||
const tabId = resolvePaneShellTabId(this.store, lease.leafId) ?? lease.tabId
|
||||
let bind: { bound: boolean; tabId: string } | null = null
|
||||
this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, {
|
||||
tabId: lease.tabId,
|
||||
leafId: lease.leafId,
|
||||
incarnationId
|
||||
})
|
||||
try {
|
||||
bind = bindPaneShell({
|
||||
store: this.store,
|
||||
this.store.persistPtyBinding({
|
||||
worktreeId: lease.worktreeId,
|
||||
tabId,
|
||||
tabId: lease.tabId,
|
||||
leafId: lease.leafId,
|
||||
ptyId: appPtyId,
|
||||
incarnationId,
|
||||
mayCreate: false
|
||||
incarnationId
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error)
|
||||
}
|
||||
if (bind?.bound === false) {
|
||||
// Unresolved, not dead: the remote shell keeps running and stays
|
||||
// reattachable once a durable pane names it again.
|
||||
console.info(
|
||||
`[ssh-relay-session] No durable pane owns ${appPtyId} on ${this.targetId}; left running unbound.`
|
||||
)
|
||||
return
|
||||
}
|
||||
// The tab the bind resolved, not the lease's frozen one — a moved pane must not be recorded
|
||||
// in the graph under the tab it left while its record and fence name the tab it is in.
|
||||
this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, {
|
||||
tabId,
|
||||
leafId: lease.leafId,
|
||||
incarnationId
|
||||
})
|
||||
return
|
||||
}
|
||||
this.runtime?.onPtySpawned(appPtyId, incarnationId, { awaitsRegistration: false })
|
||||
@@ -2639,10 +2572,9 @@ export class SshRelaySession {
|
||||
private async attachPtyWithRetry(
|
||||
ptyProvider: SshPtyProvider,
|
||||
ptyId: string,
|
||||
expectedIdentity: ExpectedPtyIdentity | undefined,
|
||||
recoveryRequest: PtySourceRecoveryRequest | undefined,
|
||||
shouldContinue: () => boolean,
|
||||
expectedIncarnationId: string | undefined,
|
||||
legacyExpectedIdentity: LegacyExpectedPtyIdentity | undefined
|
||||
shouldContinue: () => boolean
|
||||
): Promise<SshPtyAttachResult> {
|
||||
let lastError: unknown
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
@@ -2653,26 +2585,12 @@ export class SshRelaySession {
|
||||
return await this.attachPtyWithDeadline(
|
||||
ptyProvider,
|
||||
ptyId,
|
||||
recoveryRequest,
|
||||
expectedIncarnationId,
|
||||
legacyExpectedIdentity
|
||||
expectedIdentity,
|
||||
recoveryRequest
|
||||
)
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
// A proven exit is as final as an unknown id: retrying either only burns the wait before
|
||||
// the same answer. Without this the liveness-probe reap, which now answers with proof
|
||||
// rather than "not found", would no longer match and would cost a needless retry.
|
||||
if (
|
||||
!shouldContinue() ||
|
||||
isSshPtyNotFoundError(error) ||
|
||||
isSshPtyIdentityMismatchError(error) ||
|
||||
parseMatchingPtyExitedError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
ptyId,
|
||||
expectedIncarnationId
|
||||
) !== null ||
|
||||
attempt === 1
|
||||
) {
|
||||
if (!shouldContinue() || isSshPtyNotFoundError(error) || attempt === 1) {
|
||||
throw error
|
||||
}
|
||||
await this.waitForPtyReattachRetry()
|
||||
@@ -2684,9 +2602,8 @@ export class SshRelaySession {
|
||||
private async attachPtyWithDeadline(
|
||||
ptyProvider: SshPtyProvider,
|
||||
ptyId: string,
|
||||
recoveryRequest: PtySourceRecoveryRequest | undefined,
|
||||
expectedIncarnationId: string | undefined,
|
||||
legacyExpectedIdentity: LegacyExpectedPtyIdentity | undefined
|
||||
expectedIdentity: ExpectedPtyIdentity | undefined,
|
||||
recoveryRequest: PtySourceRecoveryRequest | undefined
|
||||
): Promise<SshPtyAttachResult> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let timedOut = false
|
||||
@@ -2700,17 +2617,13 @@ export class SshRelaySession {
|
||||
timer.unref?.()
|
||||
})
|
||||
try {
|
||||
const attach =
|
||||
expectedIncarnationId !== undefined || legacyExpectedIdentity !== undefined
|
||||
? ptyProvider.attachForReconnect(
|
||||
ptyId,
|
||||
recoveryRequest,
|
||||
expectedIncarnationId,
|
||||
legacyExpectedIdentity
|
||||
)
|
||||
: recoveryRequest !== undefined
|
||||
? ptyProvider.attachForReconnect(ptyId, recoveryRequest)
|
||||
: ptyProvider.attachForReconnect(ptyId)
|
||||
const attach = expectedIdentity
|
||||
? recoveryRequest
|
||||
? ptyProvider.attachForReconnect(ptyId, expectedIdentity, recoveryRequest)
|
||||
: ptyProvider.attachForReconnect(ptyId, expectedIdentity)
|
||||
: recoveryRequest
|
||||
? ptyProvider.attachForReconnect(ptyId, undefined, recoveryRequest)
|
||||
: ptyProvider.attachForReconnect(ptyId)
|
||||
const guardedAttach = attach.then((result) => {
|
||||
if (timedOut) {
|
||||
result.sourceActivationLease?.rollback()
|
||||
@@ -2737,31 +2650,40 @@ export class SshRelaySession {
|
||||
|
||||
private handlePtyReattachFailure(
|
||||
ptyId: string,
|
||||
appPtyId: string,
|
||||
pending: PendingPtyReattach,
|
||||
error: unknown,
|
||||
expectedIncarnationId: string | undefined,
|
||||
terminatedLeaseIds: Set<string>
|
||||
error: unknown
|
||||
): void {
|
||||
// An exact exit proof settles the lease; queue it for one post-worker durable retirement batch.
|
||||
// `terminated` is not the recovery-grant state, so this cannot authorize a replacement.
|
||||
|
||||
const exitProof = parseMatchingPtyExitedError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
ptyId,
|
||||
expectedIncarnationId
|
||||
)
|
||||
if (exitProof) {
|
||||
terminatedLeaseIds.add(ptyId)
|
||||
if (!isSshPtyNotFoundError(error)) {
|
||||
pending.restoreRequired = 'reattachAttemptsExhausted'
|
||||
this.wakeRecovery(pending)
|
||||
console.warn(
|
||||
`[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after bounded reattach attempts failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
)
|
||||
return
|
||||
}
|
||||
if (isSshPtyIdentityMismatchError(error)) {
|
||||
console.warn(
|
||||
`[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
)
|
||||
return
|
||||
}
|
||||
// Nothing else proves it — not even a not-found, which the relay also returns when it merely
|
||||
// cannot hand this id back — so every other failure leaves the pane detached and recoverable.
|
||||
pending.restoreRequired = 'reattachAttemptsExhausted'
|
||||
this.wakeRecovery(pending)
|
||||
console.warn(
|
||||
`[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after reattach failed: ${
|
||||
`[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`
|
||||
)
|
||||
clearProviderPtyState(appPtyId)
|
||||
deletePtyOwnership(appPtyId)
|
||||
this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired')
|
||||
const win = this.getMainWindow()
|
||||
if (win && !win.isDestroyed()) {
|
||||
win.webContents.send('pty:exit', { id: appPtyId, code: -1 })
|
||||
}
|
||||
}
|
||||
|
||||
private async sourceRecoveryRequest(
|
||||
|
||||
@@ -1473,8 +1473,6 @@ export type PreloadApi = {
|
||||
rows: number
|
||||
cwd?: string
|
||||
cwdFallback?: 'worktree'
|
||||
/** Create a shell for this pane instead of adopting the one it records; see pty.ts. */
|
||||
createFreshShellForUnreachablePane?: boolean
|
||||
env?: Record<string, string>
|
||||
envToDelete?: string[]
|
||||
command?: string
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user