diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index f89c1b11c86..fddc3964ae4 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -2,13 +2,7 @@ "schemaVersion": 1, "updatedAt": "2026-07-28", "policy": { - "maturityLevels": [ - "experimental", - "soak", - "blocking", - "accepted-gap", - "deprecated" - ], + "maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"], "blockingPromotion": { "minimumSoakRuns": 100, "minimumSoakDays": 14, @@ -29,30 +23,12 @@ "LAN direct endpoint", "Tailscale direct endpoint" ], - "platforms": [ - "ios", - "android", - "macos", - "linux", - "windows" - ], - "providers": [ - "lan", - "tailscale", - "cloud-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "lan", - "tailscale", - "cloud-relay" - ], + "platforms": ["ios", "android", "macos", "linux", "windows"], + "providers": ["lan", "tailscale", "cloud-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["lan", "tailscale", "cloud-relay"], "coverageNotes": "Deterministic TypeScript tests cover shared close-code policy, foreground retry timers, direct-winner cancellation, and concurrent LAN/Tailscale authentication. Physical iOS/Android radios, GFE, and production relay recovery remain live-test gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca-cloud/pull/96" - ], + "motivatingLinks": ["https://github.com/stablyai/orca-cloud/pull/96"], "invariant": "A foregrounded paired phone must recover from relay HOST_OFFLINE without a foreground or network-change signal, while direct recovery must select the first authenticated configured LAN or Tailscale endpoint without serial timeout delays. Backgrounding, direct success, or stop must cancel pending work, and losing probes must close without affecting the winner.", "oracle": "Inject deterministic relay close codes, random bytes, fake timers, and independently controlled direct clients. Require HOST_OFFLINE to replace any faster transport timer with one 5-15 second retry, require no retry before the selected delay, race all unique non-relay endpoints, select the first authenticated path, close every loser exactly once, and retain no retry after direct connectivity wins.", "commands": [ @@ -88,9 +64,7 @@ }, { "file": "src/shared/mobile-relay-close-codes.test.ts", - "assertions": [ - "HOST_OFFLINE maps to self-healing full-jitter recovery" - ] + "assertions": ["HOST_OFFLINE maps to self-healing full-jitter recovery"] } ], "evidenceRuns": [ @@ -153,24 +127,12 @@ "director assignment overload", "relay broker shutdown" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "cloud-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "cloud-relay" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["cloud-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["cloud-relay"], "coverageNotes": "Deterministic main-process tests cover duplicate drain notifications, full-jitter backoff, Retry-After during initial setup and drain recovery, successful recovery, and broker-close cleanup. Packaged desktop, mixed-version fleets, GFE, and production Cloud SQL remain live-test gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca-cloud/actions/runs/30223521062" - ], + "motivatingLinks": ["https://github.com/stablyai/orca-cloud/actions/runs/30223521062"], "invariant": "One relay host may have at most one assignment attempt or retry timer per recovery path. Sustained director failure must increase the retry window up to five minutes, a bounded Retry-After must be respected during initial setup and drain recovery, shutdown must cancel pending work, and recovery must activate the authoritative assigned origin.", "oracle": "Inject duplicate drain events, deterministic randomness, fake time, repeated assignment failures, a 30-second Retry-After during initial setup and drain recovery, broker close, and eventual director recovery. Count every assignment call, require 500 ms then 1,000 ms retry windows, reject duplicate fanout, require no pre-hint retry or post-close work, and prove the recovered cell becomes authoritative.", "commands": [ @@ -193,15 +155,11 @@ }, { "file": "src/main/runtime/relay/relay-http-client.test.ts", - "assertions": [ - "assignment overload preserves a bounded Retry-After hint" - ] + "assertions": ["assignment overload preserves a bounded Retry-After hint"] }, { "file": "src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts", - "assertions": [ - "initial relay setup does not retry before Retry-After expires" - ] + "assertions": ["initial relay setup does not retry before Retry-After expires"] } ], "evidenceRuns": [ @@ -505,24 +463,10 @@ "persistent terminal reattach", "update install handoff" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "Deterministic unit coverage exercises activation gating, single-instance ownership, quit policy, local/remote CLI status, headless binding persistence, local daemon identity, SSH identity transfer, and the macOS serve update handoff from staged installer through atomic bundle replacement and target-version readiness. A macOS Electron journey covers headless promotion and persistent PTY identity. A disposable locally signed Electron canary exercised real ShipIt and a temporary LaunchAgent with the compiled production supervisor; full packaged Orca and Linux/Windows serve updates remain uncollected.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/8457", @@ -702,7 +646,11 @@ "protection": "partial", "owner": "editor-runtime", "layer": "renderer-electron-contract", - "surfaces": ["Agent Session History View Log", "Monaco external-content reconciliation", "renderer crash containment"], + "surfaces": [ + "Agent Session History View Log", + "Monaco external-content reconciliation", + "renderer crash containment" + ], "platforms": ["macos", "linux", "windows"], "providers": ["local"], "coveredPlatforms": ["macos"], @@ -739,7 +687,9 @@ }, { "file": "src/renderer/src/components/editor/monaco-content-sync.undo-history.test.ts", - "assertions": ["a real Monaco read-only live-tail append leaves canUndo false while an ordinary external update remains undoable"] + "assertions": [ + "a real Monaco read-only live-tail append leaves canUndo false while an ordinary external update remains undoable" + ] }, { "file": "src/renderer/src/components/editor/MonacoEditor.content-owner.test.tsx", @@ -747,15 +697,21 @@ }, { "file": "src/renderer/src/components/editor/EditorContent.monaco-lifecycle.test.tsx", - "assertions": ["a same-pane path switch unmounts the prior outer Monaco before real mount reconciliation refreshes the stale target; the prior retained model content and undo sentinel remain untouched"] + "assertions": [ + "a same-pane path switch unmounts the prior outer Monaco before real mount reconciliation refreshes the stale target; the prior retained model content and undo sentinel remain untouched" + ] }, { "file": "src/renderer/src/components/editor/monaco-content-sync.bench.ts", - "assertions": ["with forced GC and deterministic settlement between every arm, fresh real-Monaco 9 MiB and 50 MiB models alternate 30 append and 30 replacement samples after five warmups; append p95 stays below 50/100ms and at least 2x faster"] + "assertions": [ + "with forced GC and deterministic settlement between every arm, fresh real-Monaco 9 MiB and 50 MiB models alternate 30 append and 30 replacement samples after five warmups; append p95 stays below 50/100ms and at least 2x faster" + ] }, { "file": "src/renderer/src/components/editor/monaco-content-sync.undo-retention.bench.ts", - "assertions": ["five 10 MiB read-only live-tail appends retain zero Monaco undo-service and ArrayBuffer bytes while the undoable control retains at least 50 MiB"] + "assertions": [ + "five 10 MiB read-only live-tail appends retain zero Monaco undo-service and ArrayBuffer bytes while the undoable control retains at least 50 MiB" + ] }, { "file": "tests/e2e/agent-session-log-tail-stability.spec.ts", @@ -824,17 +780,27 @@ "summary": "The production View Log journey preserved viewport, selection, Find state, renderer liveness, and forced-GC/native budgets across three watcher appends while canUndo remained false." } ], - "runtimeBudget": { "p95Seconds": 600, "scope": "local focused renderer tests plus one Electron production-journey scenario" }, - "flakeHistory": { "status": "unknown", "evidence": "New deterministic gate with local macOS passes; CI soak history is not yet available." }, - "redGreenEvidence": { "status": "complete", "evidence": "A fail-first real-Monaco test observed canUndo=true after one read-only live-tail append, and the forced-GC 50 MiB control retained 104,858,630 bytes in Monaco's undo service. After the fix the read-only arm retained zero undo-service bytes while the editable control stayed undoable. The retained Electron gate also proves each fixed watcher arm preserves viewport, selection, Find state, and non-undoability. Production builds never install its legacy setValue control." }, - "performanceBudget": { "required": true, "evidence": "Every update retrieves the model value once and performs at most one equality-or-prefix comparison; a matching append submits only the suffix. The registered Node commands require --expose-gc and --pool=threads so worker GC is available. Current p95: 9 MiB append 5.93-7.12ms versus replacement 114.09-145.89ms; 50 MiB append 26.84-34.91ms versus replacement 602.36-699.21ms. The new 50 MiB retention arm measured 104,858,630 undo-service bytes and 104,857,790 ArrayBuffer bytes for the undoable control versus zero for read-only live-tail sync. Electron forced-GC JS-heap, renderer working-set, and OS-private-memory budgets also pass." }, + "runtimeBudget": { + "p95Seconds": 600, + "scope": "local focused renderer tests plus one Electron production-journey scenario" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "New deterministic gate with local macOS passes; CI soak history is not yet available." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "A fail-first real-Monaco test observed canUndo=true after one read-only live-tail append, and the forced-GC 50 MiB control retained 104,858,630 bytes in Monaco's undo service. After the fix the read-only arm retained zero undo-service bytes while the editable control stayed undoable. The retained Electron gate also proves each fixed watcher arm preserves viewport, selection, Find state, and non-undoability. Production builds never install its legacy setValue control." + }, + "performanceBudget": { + "required": true, + "evidence": "Every update retrieves the model value once and performs at most one equality-or-prefix comparison; a matching append submits only the suffix. The registered Node commands require --expose-gc and --pool=threads so worker GC is available. Current p95: 9 MiB append 5.93-7.12ms versus replacement 114.09-145.89ms; 50 MiB append 26.84-34.91ms versus replacement 602.36-699.21ms. The new 50 MiB retention arm measured 104,858,630 undo-service bytes and 104,857,790 ArrayBuffer bytes for the undoable control versus zero for read-only live-tail sync. Electron forced-GC JS-heap, renderer working-set, and OS-private-memory budgets also pass." + }, "promotionCriteria": [ "Collect stable soak history on macOS, Linux, and Windows.", "Accumulate 100 consecutive deterministic gate passes or 14 days without unexplained flakes." ], - "knownGaps": [ - "No live Windows or Linux View Log evidence yet." - ], + "knownGaps": ["No live Windows or Linux View Log evidence yet."], "demotionRule": "Quarantine the Electron scenario if it flakes without a product or harness bug; demote if viewport/Find drift, renderer loss, p95 regression, or memory retention exceeds the registered budgets." }, { @@ -844,23 +810,10 @@ "protection": "partial", "owner": "terminal-runtime", "layer": "renderer-unit", - "surfaces": [ - "terminal lifecycle", - "dead-session reconciliation", - "tab creation" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon" - ], - "coveredPlatforms": [ - "macos" - ], + "surfaces": ["terminal lifecycle", "dead-session reconciliation", "tab creation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the reconcile guards that exist on main@1282f5c2d. Broader targeted-hasPty resume paths, no-hot listing counts, and live Electron survival arrive with the pending reliability stack.", "motivatingLinks": [ @@ -940,31 +893,12 @@ "Manage Sessions", "Resource Manager" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime", - "mobile-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime", "mobile-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "Local macOS deterministic evidence covers the renderer snapshot/coordinator, exact local and SSH-shaped PTY request settlement, active-last and pinned terminal-tab routing, component-unmount continuation, and the existing current/legacy daemon management contract. Windows Electron process absence, live SSH/WSL behavior, and remote-runtime/mobile host completion remain explicit gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/8001" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/8001"], "invariant": "Every terminal surface confirmed in the invoking renderer is force-closed exactly once after daemon management settles, later-created surfaces and non-terminal tabs survive, and exact shutdown requests are limited to deduplicated current non-runtime PTY bindings of the confirmed surfaces.", "oracle": "Snapshot terminal entity IDs before the first await; mutate ownership, active selection, bindings, and tab presence while daemon management is pending and between bounded close batches; then assert only the immutable targets disappear from both terminal stores, active targets close last with valid editor/browser/deactivated post-state, every captured exact PTY promise settles before callbacks, and no provider inventory sweep or late-tab kill occurs.", "commands": [ @@ -1067,26 +1001,10 @@ "hidden terminal parking", "agent resume authority" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh", - "runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh", "runtime"], "coverageNotes": "Live macOS Electron tests prove exact local PTY disappearance after parked-tab close and detached-pgid descendant death after agent close. Deterministic tests cover daemon and SSH routing, local/daemon pending-snapshot ownership across natural exit, stale-root descendant-signal suppression, graceful-to-immediate kill upgrades, duplicate-kill completion sharing, locale-stable bounded/fresh/coalesced process-table reads, deadline-safe successor scans, cycle-safe linear descendant traversal, target-only escalation indexing, 32-wide bulk teardown, source-scan timestamp identity, same-second PID ambiguity, ordinary runtime close ownership, unified-only hydration, split ownership, pane detach transfer, restart alias hydration, and late-hook suppression; live Linux, Windows, WSL, SSH, and remote-runtime process evidence remains pending.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/8628", @@ -1329,9 +1247,7 @@ }, { "file": "src/main/runtime/rpc/e2ee-channel-v2.test.ts", - "assertions": [ - "mobile E2EE v2 continues to reject additive runtime capability metadata" - ] + "assertions": ["mobile E2EE v2 continues to reject additive runtime capability metadata"] }, { "file": "src/main/runtime/rpc/runtime-client-capabilities.test.ts", @@ -1459,33 +1375,11 @@ "protection": "partial", "owner": "terminal-runtime", "layer": "shared-main-renderer-contract", - "surfaces": [ - "terminal lifecycle", - "tab creation", - "PTY spawn", - "startup cwd persistence" - ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "ssh", - "remote-runtime" - ], + "surfaces": ["terminal lifecycle", "tab creation", "PTY spawn", "startup cwd persistence"], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "ssh", "remote-runtime"], "coverageNotes": "Local macOS evidence covers the shared missing-dir fallback policy, main pty:spawn recovery and metadata, no-flag and reattach strictness, renderer IPC flag routing, SSH-tagged and remote-runtime omission, and the visibility-gated terminal notice. Daemon shares the same pre-provider main cwd decision but lacks a live daemon-provider run; WSL UNC paths are exempt from the probe by design and lack a live run; Linux/Windows and mobile/API strictness are gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/7239", @@ -1622,21 +1516,9 @@ "runtime reload and session replacement", "loopback restart and stall recovery" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS execution of the generated Pi/OMP extension plus the shared hook normalizer. WSL fallback behavior is covered with mocked native-fetch failure and Windows curl handoff. Daemon PTYs use the same generated extension without a distinct delivery path. SSH/relay ingest uses the shared normalizer, but no live remote Pi process is exercised.", "motivatingLinks": [ @@ -1726,21 +1608,9 @@ "provider session dedupe", "sidebar and mobile identity" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the ownership/dedupe suite on main@1282f5c2d. Queued/pending resume-claim indexing, same-session and wrong-session hook proofs, and Electron repeat-activation coverage arrive with the pending stack (#7008).", "motivatingLinks": [ @@ -1754,9 +1624,7 @@ "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/resume-sleeping-agent-session.test.ts" ], - "testFiles": [ - "src/renderer/src/lib/resume-sleeping-agent-session.test.ts" - ], + "testFiles": ["src/renderer/src/lib/resume-sleeping-agent-session.test.ts"], "assertionRefs": [ { "file": "src/renderer/src/lib/resume-sleeping-agent-session.test.ts", @@ -1825,28 +1693,10 @@ "terminal exit retirement and restart restore", "mixed-version fallback" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coverageNotes": "Deterministic macOS tests cover controller claims, daemon and SSH/relay operation replay, mixed-version selection, runtime ownership, exact provisional handoff, durable terminal retirement, two independent viewer mirrors, guarded adoption of legacy live PTYs, and completion classification when either the outer remote transport or authoritative host/provider process inspection becomes unreachable. The adoption harness models v1.4.150 agent/setup/shell tabs, current-generation restart and reconnect, exact handle/incarnation/worktree/host checks, topology CAS, competing clients, split-pane/group restoration, WSL ownership, and SSH owner rejection. The secondary parity repro runs independent clients against one headless remote Orca runtime over encrypted pairing and a real daemon-backed PTY, with tokened fixture-process identity separated from unrelated Codex app-server startup probes. The automated primary topology runs an isolated headed macOS Orca desktop server plus a separate paired web client and proves viewer-local fresh/resume focus, exact legacy placement, writable PTYs, unrelated-terminal survival, and host/client cleanup. SSH coverage is provider/relay contract and fault-injection coverage only; it does not substitute for paired-server coverage. Live Windows, Linux, WSL, SSH, and physical paired-Linux hosts remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/8878", @@ -1935,9 +1785,7 @@ }, { "file": "src/relay/pty-handler.test.ts", - "assertions": [ - "strict relay inspection rejects a missing PTY" - ] + "assertions": ["strict relay inspection rejects a missing PTY"] }, { "file": "tests/e2e/remote-agent-completion-authority.unit.test.ts", @@ -2218,31 +2066,12 @@ "browser and mobile handoff", "app restart" ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "local", - "remote-runtime", - "ssh", - "wsl" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "remote-runtime", - "ssh", - "wsl" - ], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["local", "remote-runtime", "ssh", "wsl"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "remote-runtime", "ssh", "wsl"], "coverageNotes": "Platform-neutral deterministic tests separate the durable Active Server preference from per-client connection, selected-workspace, browser-session, and execution-host routing. The composed regression models Local desktop -> connect/navigate Windows 2 -> reveal a local terminal -> restart. Multi-client browser host overrides, multi-server profile caches, generic settings IPC rejection, local and remote workspace ownership, and restart reset of transient routing are covered. Live desktop UI runs remain uncollected.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/9687" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/9687"], "invariant": "Only an explicit user change in Settings > Remote Orca Servers > Advanced > Active Server may mutate activeRuntimeEnvironmentId. Connecting, pairing, disconnecting, selecting or revealing a workspace or terminal, browser/mobile handoff, remote navigation, and reconnect must use transient or target-owner routing and must never rewrite the durable preference. Generic settings mutation cannot bypass the dedicated preference IPC.", "oracle": "Start with Active Server=Local desktop, connect and navigate Windows 2, then reveal a local terminal and assert it succeeds while the persisted preference remains local before and after restart. Repeat with multiple clients and servers, browser host switches, remote-owned and local-owned workspaces, pairing/connect/disconnect, and generic settings writes. Assert only the dedicated validated preference method changes activeRuntimeEnvironmentId and stale host-operation completions cannot overwrite the newly selected transient host.", "commands": [ @@ -2355,20 +2184,9 @@ "hidden-to-visible transitions", "window wake" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence on main@1282f5c2d, including #7192's runtime-mirror geometry authority slice. Deterministic provider-contract coverage now includes settled window-wake reassertion and SSH relay applied-size readback. Live shell-visible SSH/remote geometry and Windows ConPTY readback remain non-blocking gaps.", "motivatingLinks": [ @@ -2502,27 +2320,10 @@ "protection": "partial", "owner": "terminal-rendering", "layer": "renderer-unit", - "surfaces": [ - "terminal search", - "links", - "WebGL", - "decorations", - "keyboard navigation" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "surfaces": ["terminal search", "links", "WebGL", "decorations", "keyboard navigation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the WebGL/link/search containment suites on main@1282f5c2d, adopting the #6949 atlas-recovery rename and #7133's reveal hardening tests. Core addon-load throw containment and live typed-input survival arrive with #7004 and a live follow-up.", "motivatingLinks": [ @@ -2646,34 +2447,18 @@ "xterm scrollbar DOM", "scrollback" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Renderer-unit coverage proves the shared xterm DOM intent path. Live Electron evidence is PR validation evidence for local macOS only until the flow has stable automation; live SSH, WSL, Linux, and Windows paths remain unproved.", - "motivatingLinks": [ - "STA-1341" - ], + "motivatingLinks": ["STA-1341"], "invariant": "A user-driven xterm scrollbar thumb or track scroll updates the live terminal scroll intent before tab, visibility, or layout resume enforces intent, so resume preserves the latest dragged viewport instead of an older pinned line.", "oracle": "Pointerdown on .xterm-scrollbar or .xterm-slider followed by xterm viewport movement records the new pinned viewport, and enforcing current intent restores that dragged line instead of stale top intent.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts" ], - "testFiles": [ - "src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts" - ], + "testFiles": ["src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts"], "assertionRefs": [ { "file": "src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts", @@ -2730,23 +2515,9 @@ "protection": "none", "owner": "startup-persistence", "layer": "upgrade-fixture", - "surfaces": [ - "startup", - "upgrade", - "session restore", - "daemon restore" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl" - ], + "surfaces": ["startup", "upgrade", "session restore", "daemon restore"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -2782,9 +2553,7 @@ "Run second restart after current code writes upgraded state.", "Record startup timing and failure artifact." ], - "knownGaps": [ - "No fixture corpus or command yet." - ], + "knownGaps": ["No fixture corpus or command yet."], "demotionRule": "Cannot promote without old production fixture provenance." }, { @@ -2803,13 +2572,8 @@ "resize", "exit cleanup" ], - "platforms": [ - "linux", - "macos" - ], - "providers": [ - "local" - ], + "platforms": ["linux", "macos"], + "providers": ["local"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The live Electron Playwright slice exists only on the pending reliability stack. It registers here with its owning split PR.", @@ -2960,20 +2724,10 @@ "CJK repaint", "cursor and resize" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], - "coveredPlatforms": [ - "windows" - ], - "coveredProviders": [ - "daemon" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], + "coveredPlatforms": ["windows"], + "coveredProviders": ["daemon"], "coverageNotes": "Issue #8048 now has deterministic wrapper and cold-restore re-anchor tests plus a Windows PR-CI harness that drives the built daemon through 25 real ConPTY workspace-close races while an unrelated witness PTY stays alive. Keyboard reset, CJK repaint, WSL, and full visible Electron coverage remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6541", @@ -3167,21 +2921,9 @@ "resize", "render" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Platform-neutral unit coverage proves the Resource Manager closed badge performs one readiness seed, coalesces unknown spawn signals, skips known-session reattach signals, and installs no interval. Broader terminal interaction coverage remains on the pending reliability stack.", "motivatingLinks": [ @@ -3217,9 +2959,7 @@ }, { "file": "src/main/ipc/pty.test.ts", - "assertions": [ - "global inventory starts local and SSH provider listings concurrently" - ] + "assertions": ["global inventory starts local and SSH provider listings concurrently"] }, { "file": "src/renderer/src/components/status-bar/resource-session-inventory.test.ts", @@ -3289,18 +3029,8 @@ "provider ownership", "diagnostics bundle" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The crash-breadcrumb recording and its test exist only on the pending reliability stack. It registers here with its owning split PR.", @@ -3355,40 +3085,33 @@ "main PTY batching", "runtime path provenance", "runtime terminal wait detection", + "SSH relay frame decoding", + "SSH PTY source retention", + "SSH PTY reconnect waves", "renderer ACK", "xterm scheduler", "hidden output" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "ssh" - ], - "coverageNotes": "Local macOS evidence covers main-process pending-output caps, runtime path-provenance history reuse, and saturated-tail wait detection. A macOS-hosted Docker OpenSSH run exercises the deployed Linux relay and SSH provider with an exact 256 KiB source-credit plateau, concurrent PTY typing, fixed-size filesystem frames, Git churn, and owner-lease reconnect. It does not cover local/daemon, paired remote-runtime, headed desktop, headless orca serve, prior-version daemons, WSL, Windows named pipes/ConPTY, folder workspaces, or mixed-version clients.", + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["ssh"], + "coverageNotes": "Local macOS evidence covers main-process pending-output caps, runtime path-provenance history reuse, saturated-tail wait detection, direct-SSH source retention/accounting, decoder input bounds, and bounded reconnect scheduling through deterministic unit contracts. A macOS-hosted Docker OpenSSH run exercises the deployed Linux relay and SSH provider with an exact 256 KiB source-credit plateau, concurrent PTY typing, fixed-size filesystem frames, Git churn, and owner-lease reconnect. It does not cover local/daemon, paired remote-runtime, headed desktop, headless orca serve, prior-version daemons, WSL, Windows named pipes/ConPTY, folder workspaces, or mixed-version clients.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6836", "https://github.com/stablyai/orca/pull/6858", "https://github.com/stablyai/orca/pull/7002", "https://github.com/stablyai/orca/pull/7054" ], - "invariant": "High-volume terminal output must stay bounded across daemon socket writes, main runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input.", - "oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, and ACK-gated in-flight bounds. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.", + "invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through exit, cancellation, recovery, desktop projection, and required remote replacement.", + "oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, and ACK-gated in-flight bounds. Direct-SSH contracts additionally prove one-write ownership across write(false)/drain, liveness rebasing, source-obligation terminality before ACK publication, immutable desktop/remote mappings with rollback, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime-path-candidate-history.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/orca-runtime-tail-wait-memo.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot", "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" ], "testFiles": [ @@ -3396,6 +3119,29 @@ "src/main/runtime/orca-runtime-path-candidate-history.test.ts", "src/main/runtime/orca-runtime.test.ts", "src/main/runtime/orca-runtime-tail-wait-memo.test.ts", + "src/main/providers/ssh-pty-provider.test.ts", + "src/main/ssh/ssh-relay-session.test.ts", + "src/main/ssh/ssh-relay-session-data-delivery.test.ts", + "src/main/ssh/ssh-relay-session-recovery-races.test.ts", + "src/main/ssh/ssh-relay-session-incarnation.test.ts", + "src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts", + "src/main/ssh/ssh-relay-session-terminal-error.test.ts", + "src/main/ssh/ssh-pty-recovery-retention-budget.test.ts", + "src/main/ssh/relay-protocol-backpressure.test.ts", + "src/relay/protocol-backpressure.test.ts", + "src/relay/pty-source-credit-ledger.test.ts", + "src/relay/pty-source-credit-scheduler.test.ts", + "src/relay/relay-pty-source-publication.test.ts", + "src/relay/ssh-pty-source-credit-adapter.test.ts", + "src/shared/pty-consumer-session.test.ts", + "src/main/ipc/ssh-pty-output-intake.test.ts", + "src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts", + "src/main/runtime/rpc/terminal-multiplex.test.ts", + "src/main/ssh/ssh-multiplexer-transport-writer.test.ts", + "src/main/ssh/ssh-channel-multiplexer.test.ts", + "src/main/ssh/ssh-pty-source-credit-rollout.test.ts", + "src/main/ssh/ssh-relay-deploy.test.ts", + "src/relay/dispatcher-client-writer.test.ts", "tests/e2e/ssh-docker-relay-perf.spec.ts" ], "assertionRefs": [ @@ -3431,6 +3177,98 @@ "resolves tui-idle when a stale Codex prompt is followed by the ready header" ] }, + { + "file": "src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts", + "assertions": [ + "fifty reattaches use at most eight workers and healthy siblings finish before one slow and one failed PTY" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session.test.ts", + "assertions": [ + "each transient PTY failure is retried once without tearing down successful provider registration" + ] + }, + { + "file": "src/main/ssh/ssh-pty-recovery-retention-budget.test.ts", + "assertions": [ + "fragmented recovery is bounded per PTY and per session in source units, charged bytes, and frames", + "UTF-16 string storage and record overhead are charged before aggregate admission" + ] + }, + { + "file": "src/relay/pty-source-credit-ledger.test.ts", + "assertions": [ + "source retention charges max UTF-8 or UTF-16 storage plus one record", + "fragmented multi-PTY source, charged-byte, and frame caps bind independently" + ] + }, + { + "file": "src/main/ssh/relay-protocol-backpressure.test.ts", + "assertions": [ + "main SSH decoder accepts one maximum frame plus 1 MiB partial input and rejects one extra byte" + ] + }, + { + "file": "src/relay/protocol-backpressure.test.ts", + "assertions": [ + "relay decoder accepts one maximum frame plus 1 MiB partial input and rejects one extra byte" + ] + }, + { + "file": "src/main/ipc/ssh-pty-output-intake.test.ts", + "assertions": [ + "exit remains behind accepted model, desktop projection, and required remote obligations", + "exit timeout cancels only the matching delivery and keeps unrelated provider work usable", + "desktop source identity and scanner facts commit atomically or roll back without credit" + ] + }, + { + "file": "src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts", + "assertions": [ + "headed and headless replacement commits only after current-generation SnapshotEnd sequence coverage", + "failed or stale replacement rolls back to the live stream obligation", + "late detach and replacement commit prune spans already reclaimed by cancellation proof" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session-data-delivery.test.ts", + "assertions": [ + "stale owner recovery clears cached state and retries exactly once without resume", + "recovery remains quarantined until exact checkpoint-to-recoveryEnd coverage and live handoff", + "invalid recovery cancels only its replacement token without physical PTY or ownership teardown" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session-recovery-races.test.ts", + "assertions": [ + "empty recovery retains recoveryEndSu as the first live-frame continuity anchor", + "validated token cancellation drops queued late frames without physical PTY teardown", + "overlapping recovery never cancels on or mutates the replacement mux, checkpoint, lease, provider state, or ownership" + ] + }, + { + "file": "src/main/ssh/ssh-multiplexer-transport-writer.test.ts", + "assertions": [ + "write(false) owns one frame and later ordinary traffic waits for drain", + "one coalesced liveness bypass is allowed per saturated epoch", + "saturation and drain transitions are reported exactly once" + ] + }, + { + "file": "src/main/ssh/ssh-channel-multiplexer.test.ts", + "assertions": [ + "self-imposed writer saturation suppresses false death and rebases both health clocks on drain" + ] + }, + { + "file": "src/relay/dispatcher-client-writer.test.ts", + "assertions": [ + "encoded producer frames stay below high-water-minus-reserve capacity", + "one fixed-size filesystem compatibility frame is admitted only on an empty sink", + "control remains reserved while producer retention stays bounded" + ] + }, { "file": "tests/e2e/ssh-docker-relay-perf.spec.ts", "assertions": [ @@ -3468,6 +3306,15 @@ "durationSeconds": 21.33, "summary": "2 test files passed, 609 tests passed on pushed commit 4fb14eac3897." }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 1.96, + "summary": "13 deterministic SSH relay/session/source/decoder unit files passed with 161 tests; no live topology was exercised." + }, { "date": "2026-07-27", "runner": "local", @@ -3484,11 +3331,11 @@ }, "flakeHistory": { "status": "unknown", - "evidence": "Focused main-process backlog tests are deterministic unit slices. On 2026-07-27 the source-credit reconnect case passed once in the full four-test Docker suite (15.7 seconds) and three consecutive isolated repeat-each runs (21.1, 19.8, and 15.3 seconds); the combined perf soak still needs longer runtime history before promotion." + "evidence": "Focused main-process backlog and direct-SSH reconnect/memory/decoder tests are deterministic unit slices. On 2026-07-27 the source-credit reconnect case passed once in the full four-test Docker suite (15.7 seconds) and three consecutive isolated repeat-each runs (21.1, 19.8, and 15.3 seconds); the combined perf soak still needs longer runtime history before promotion." }, "redGreenEvidence": { "status": "partial", - "evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output is capped per PTY and in total, total-pressure trimming prefers background pending output before active pending output, trimmed pending tails preserve seq/rawLength metadata, and ACK-gated in-flight output remains bounded. Intentionally restoring the old path-history shape failed the focused scale test after 774ms and 4,096 replacements; disabling the ordinary-tail branch failed with a retained 253,999-character waitText. The fixed shapes passed their focused suites. Existing renderer tests cover replay/backlog slices. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion." + "evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output is capped per PTY and in total, direct-SSH source/recovery memory and both frame decoders stay within exact caps, and 50-PTY reconnect isolates slow/failing siblings. The new controls failed before the fixes with serial reattach, UTF-8-only accounting, and roughly 32 MiB decoder retention. Existing renderer tests cover replay/backlog slices. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion." }, "performanceBudget": { "required": true, @@ -3525,15 +3372,8 @@ "provider ownership", "startup restore" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "daemon", - "local" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["daemon", "local"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The fail-closed degraded-daemon hardening and its contracts exist only on the pending reliability stack. It registers here with its owning split PR.", @@ -3589,24 +3429,10 @@ "runtime terminal stop", "restored terminal teardown" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "A real daemon server over a local socket proves a fresh adapter can kill a live session before any prior client operation. Main-process tests prove renderer IPC, runtime kill, and runtime exact-stop wait for the provider swap and issue zero shutdowns to the fallback provider, while SSH spawn and kill bypass the local barrier. The same shared logic runs on Linux and Windows; live platform runs remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/7742", @@ -4201,12 +4027,8 @@ "provider liveness", "restore" ], - "platforms": [ - "windows" - ], - "providers": [ - "wsl" - ], + "platforms": ["windows"], + "providers": ["wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -4264,17 +4086,8 @@ "metadata-only replay", "WebGL recovery" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The replay FIFO/burst coalescing product change and its tests exist only on the pending reliability stack; main still uses a single pendingReplayData slot. It registers here with its owning split PR.", @@ -4333,21 +4146,9 @@ "keyboard bypass", "JIS yen" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos", - "linux" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos", "linux"], "coveredProviders": [], "coverageNotes": "Local macOS and containerized Linux evidence, deterministic renderer-unit coverage for the Linux/Sogou candidate-key policy including the legacy orphaned-keyup fallback, and Electron/CDP live-PTY Sogou-style repros. Real Linux/Sogou OS IME automation, Windows ConPTY post-agent reset, and the CJK/Vietnamese/Arabic matrix remain registered gaps.", "motivatingLinks": [ @@ -4390,9 +4191,7 @@ }, { "file": "src/renderer/src/components/terminal-pane/terminal-paste-runtime.test.ts", - "assertions": [ - "paste/runtime forwarding avoids duplicate terminal payloads" - ] + "assertions": ["paste/runtime forwarding avoids duplicate terminal payloads"] }, { "file": "src/renderer/src/components/terminal-pane/terminal-ime-composition-tracker.test.ts", @@ -4529,14 +4328,8 @@ "TUI exit", "standard key input" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -4593,21 +4386,9 @@ "WSL", "remote runtime" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Deterministic main/renderer tests run on macOS and exercise simulated Windows plus POSIX/Linux process-confirmation behavior. They cover exact local-ConPTY membership, detached-child rejection, SSH and paired-runtime host-platform routing including legacy runtime PTY IDs, active-PTY ownership after worktree host changes, Windows-to-WSL routing, unknown-metadata fallback, production handler composition, KKP authorization on every host, process-confirmed Droid routing and launch-triggered confirmation, typed-alias and forged-OSC isolation, shells without OSC 133, split/detach ownership with preserved shell classification, stale PTY exit/rebind rejection, command-generation revocation, daemon v21 warm-reattach identity, unavailable inspection, and lazy subprocess/RPC callback counts. A live linux-arm64 Docker target additionally proved real relay deployment, SSH PTY host detection, and exact inactive/active KKP bytes. The Electron byte test runs only on Windows; live Windows evidence remains uncollected. Test failures report exact expected bytes; no product telemetry or raw terminal logging is added.", "motivatingLinks": [ @@ -4868,29 +4649,12 @@ "PTY identity", "foreground process confirmation" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon"], "coverageNotes": "Runtime and provider-contract tests on macOS cover exact-PTY authorization, local/daemon fresh confirmation, exact ConPTY membership, and unsupported-provider fail-closed behavior. Physical Windows, live Linux, SSH, WSL, and remote-runtime validation remain explicit gaps; providers without confirmation preserve conservative refusal on a shell conflict.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/8303" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/8303"], "invariant": "A guarded note send writes only to the exact PTY binding checked by the guard and only while permission/wait evidence allows input; fresh hook state conflicting with an ordinary shell foreground requires fresh provider confirmation of a recognized agent in that PTY.", "oracle": "Fresh explicit state plus ordinary PowerShell plus confirmed recognized agent is sendable on the same PTY. Confirmed shell/non-agent, unavailable confirmation, PTY exit, handle rebind, or a callback PTY mismatch returns a refusal or not-writable result and writes zero bytes.", "commands": [ @@ -4942,9 +4706,7 @@ }, { "file": "src/main/providers/local-pty-provider.test.ts", - "assertions": [ - "fresh confirmation is discarded when its owning local PTY exits" - ] + "assertions": ["fresh confirmation is discarded when its owning local PTY exits"] }, { "file": "src/main/providers/windows-conpty-process-membership.test.ts", @@ -4967,9 +4729,7 @@ }, { "file": "src/renderer/src/lib/active-agent-note-send.test.ts", - "assertions": [ - "selected active-agent note sends retain guarded paste and submit routing" - ] + "assertions": ["selected active-agent note sends retain guarded paste and submit routing"] }, { "file": "src/renderer/src/components/browser-pane/BrowserAnnotationSendMenuContent.test.tsx", @@ -5033,27 +4793,12 @@ "PTY writes", "bracketed paste" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local"], "coverageNotes": "Local macOS evidence covers the runtime PTY write contract and a live dev-runtime CLI repro. SSH, daemon, remote-runtime, Linux, and Windows remain provider/platform gaps; the product path stays provider-owned and does not add local filesystem or process assumptions.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/7226" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/7226"], "invariant": "Injected orchestration task prompts for recognized agent CLIs must send the prompt body inside one bracketed-paste frame, sanitize embedded ESC bytes, preserve chunk boundaries without losing the frame, and send Enter only after the paste frame completes.", "oracle": "Runtime tests assert the exact PTY write sequence and failure cleanup; orchestration tests assert dispatch/coordinator use the agent prompt path; the live CLI harness dispatches a 32KB task to a fake Codex-like TUI and requires marker present, bracketed paste present, zero unframed line breaks, and submit observed.", "commands": [ @@ -5313,14 +5058,8 @@ "cursor repaint", "rewrite output" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -5356,10 +5095,7 @@ "Keep stress cases non-blocking until Windows runtime history is stable.", "Fail promotion on silent Windows environment skips." ], - "knownGaps": [ - "No manifest command yet.", - "No Windows CJK/emoji repaint command is wired." - ], + "knownGaps": ["No manifest command yet.", "No Windows CJK/emoji repaint command is wired."], "demotionRule": "Cannot promote if the oracle is screenshot-only or environment-skipped." }, { @@ -5377,14 +5113,8 @@ "local provider", "daemon provider" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -5439,17 +5169,8 @@ "renderer CPU", "resize churn" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -5497,21 +5218,9 @@ "protection": "none", "owner": "terminal-performance", "layer": "daemon-provider-contract", - "surfaces": [ - "daemon stream", - "socket write", - "drain", - "hidden output", - "input starvation" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "daemon" - ], + "surfaces": ["daemon stream", "socket write", "drain", "hidden output", "input starvation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["daemon"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The daemon batcher write(false)/drain contracts exist only on the pending reliability stack. It registers here with its owning split PR.", @@ -5569,23 +5278,12 @@ "workspace switch", "agent status" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The boot-hydration counters and their tests exist only on the pending reliability stack. It registers here with its owning split PR.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/7002" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/7002"], "invariant": "Terminal typing, focus, resize, tab/workspace switch, and agent/session restore must not trigger unbounded store projection, git status, provider listing, or per-pane polling work.", "oracle": "The current executable slice instruments boot-time local PTY registry hydration with repo counts, local-vs-remote repo skips, worktree enumeration counts, adapter/session listing counts, registration/skipped-session counts, duration, and failure phase. The broader oracle still needs instrumentation that counts store selector recomputes, git status requests, provider listings, and session scans during scripted hot interactions with many worktrees and terminal panes.", "commands": [], @@ -5634,20 +5332,9 @@ "metadata-only replay", "clear semantics" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the merged #7133/#7173 restore and hidden-output ordering tests on main@1282f5c2d. The dirty-state exactness contract, normal-buffer clear semantics, and metadata-only replay remain pending-stack work.", "motivatingLinks": [ @@ -5662,9 +5349,7 @@ "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts" ], - "testFiles": [ - "src/renderer/src/components/terminal-pane/pty-connection.test.ts" - ], + "testFiles": ["src/renderer/src/components/terminal-pane/pty-connection.test.ts"], "assertionRefs": [ { "file": "src/renderer/src/components/terminal-pane/pty-connection.test.ts", @@ -5732,23 +5417,12 @@ "renderer stream", "terminal colors" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/6949" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/6949"], "invariant": "Startup OSC 10/11 color queries are answered out of band at startup only, never leak into shell/provider output streams, and ordinary runtime OSC color queries remain renderer-handled.", "oracle": "A provider-contract fixture records startup query replies, shell-visible bytes, renderer-visible bytes, and later runtime OSC behavior to prove no query leakage or color deadlock.", "commands": [], @@ -5776,10 +5450,7 @@ "Cover startup-only and runtime OSC paths separately.", "Keep screenshot evidence diagnostic only." ], - "knownGaps": [ - "No manifest command yet.", - "No startup color-query contract is wired." - ], + "knownGaps": ["No manifest command yet.", "No startup color-query contract is wired."], "demotionRule": "Cannot promote if success is based only on absence of visible artifacts." }, { @@ -5796,21 +5467,9 @@ "mobile subscription replay", "multi-mobile input floor" ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Deterministic local tests execute the exact injected mobile replay/generation gate, the React Native query classifier, stale-subscription sender, server-side single-responder election, query-reply RPC semantics, and live-output capture during async mobile fit. The provider write path is shared, but no live iOS/Android, SSH, WSL, or multi-device run is registered yet.", "motivatingLinks": [ @@ -5942,21 +5601,9 @@ "JSON subscribe fallback", "snapshot buffering" ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "remote-runtime", - "ssh", - "local", - "daemon" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["remote-runtime", "ssh", "local", "daemon"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence covers runtime-RPC stream budgets plus paired-renderer parse/discard credit. Deferred credit is shared by local and remote transports, batches ACKs at 192 KiB or 4 ms, grows per-stream windows from 512 KiB to 2 MiB and aggregate windows from 2 MiB to 8 MiB, bounds queued output to 256 KiB per stream, and caps each multiplex connection at 32 active or pending streams for an 8 MiB aggregate pending-output ceiling. Deterministic tests cover replay ordering, stale generations, malformed frames, hidden panes, queue eviction, disposal, send/recovery failure, repeated pending-slot replacement, reconnect, and round-robin fairness. The opt-in benchmark covers 1/20/100 ms RTT and 1/4/8 viewers, exact protocol-frame allocations, scheduler CPU, and measured @xterm/headless parser CPU/retained heap. Live Android restore evidence, browser/WebGL parser measurements, and legacy JSON subscribe parity remain required.", "motivatingLinks": [ @@ -6124,20 +5771,9 @@ "unicode width", "parser configuration" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over #7148's width-parity oracle on main@1282f5c2d. A broader byte corpus, a shared parser-construction assertion over terminal-unicode-provider.ts/pane-terminal-options.ts, and recorded agent-session corpora remain gaps.", "motivatingLinks": [ @@ -6149,9 +5785,7 @@ "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/headless-emulator-unicode-width.test.ts" ], - "testFiles": [ - "src/main/daemon/headless-emulator-unicode-width.test.ts" - ], + "testFiles": ["src/main/daemon/headless-emulator-unicode-width.test.ts"], "assertionRefs": [ { "file": "src/main/daemon/headless-emulator-unicode-width.test.ts", @@ -6208,23 +5842,9 @@ "protection": "none", "owner": "terminal-rendering", "layer": "renderer-observability", - "surfaces": [ - "hidden-output restore", - "snapshot replay", - "anomaly breadcrumbs", - "telemetry" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "surfaces": ["hidden-output restore", "snapshot replay", "anomaly breadcrumbs", "telemetry"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; the anomaly-breadcrumb machinery exists in this branch but no convergence probe is implemented.", @@ -6349,15 +5969,8 @@ "window wake", "render model" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; the live repro harness with a refresh-repair oracle exists but is not productized into the release-blocking terminal-rendering-golden suite.", @@ -6411,18 +6024,9 @@ "renderer reload / crash recovery", "surviving daemon and local PTYs" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS unit evidence over the shared main-process delivery pipeline in registerPtyHandlers. The oracle drives a local mock PTY, but the in-flight/pending accounting is provider-agnostic and daemon PTYs ride the identical pipeline (the manual dev repro that proved the freeze was daemon-backed). SSH is a separate relay/credit path and never enters these counters; mobile/relay ride unaffected paths; WSL terminals ride this same local/daemon pipeline and are covered by the same accounting (no dedicated provider-contract test yet). Motivated by a production incident on v1.4.131 (three frozen/broken terminal panes in one desktop session) diagnosed to leaked in-flight/pending accounting across a renderer reload (rendererGraphEpoch 3). Live validation of the fix on a rebuilt dev instance surfaced a second leak mode (the boot window): after the reset ran, main resumed flushing a flooding PTY into the still-booting page before its pty:data listener re-registered, so those sends were dropped yet counted and re-pinned rendererInFlightChars at 524288. The gate now also covers holding sends until the renderer's pty:rendererDispatcherReady handshake. A watchdog self-heals a lost handshake: on each reset a one-shot ~10s timer arms and, if the handshake never arrives, force-opens the gate (rendererDispatcherReadyForcedCount increments) so a dropped handshake degrades to pre-handshake behavior instead of a permanent hold; the real handshake or a re-registration cancels it. Two new diagnostics — rendererPtyDispatcherReady and rendererDispatcherReadyForcedCount — expose the otherwise-invisible boot-window hold, which early-returns before ackGatedFlushSkipCount increments.", "motivatingLinks": [ @@ -6516,24 +6120,10 @@ "desktop filesystem watcher", "SSH relay filesystem watcher and live PTYs" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "remote-runtime", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "remote-runtime", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "remote-runtime", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "remote-runtime", "ssh"], "coverageNotes": "Deterministic tests cover one shared healthy child per runtime process plus at most four bounded fault-quarantine children, native desktop, paired-runtime, and WSL snapshot processes sharing the same eight-physical-child reservation with typed serialized event-driven capacity recovery, including recursive re-wait when crash recovery reclaims an announced slot, one quarantine attempt per watch lifetime, generation-scoped cancellation, bounded termination with removable deadline waiters, replacement-crawl errors that cannot be blessed by a late readiness ack, pre-ready paired-web cancellation registered before an unbounded capacity wait, already-resolved and late physical-exit retry of a rejected paired-web teardown, physical desktop-install and local/SSH PTY teardown, stale-generation rejection, per-child stat bounds, final-overflow RPC delivery, and renderer eviction before terminal callbacks can retry. Destructive local and SSH removal fences both ID-derived and resolved-cwd terminal roots, including sibling-root/cwd combinations with reverse admission rollback, closes descendant relay watches before parent deletion, enumerates authoritative provider/cwd ownership, falls back to daemon spawn cwd before OSC 7, and keeps daemon/relay immediate shutdown pending until native exit or a bounded fail-closed error; a dead relay PID discovered during attach also settles concurrent shutdown before its stale entry is reaped, while Windows runtime watcher deadlines retain one removable physical-close waiter, treat Node's error-close path as positive physical-exit proof, and clear root ownership on late close. The relay policy test maps a standard repository's base, Git common directory, and worktree roots to one healthy supervisor, then proves a shared-child failure recovers those roots in separate quarantine supervisors with overflow and resumed events. Abortable capacity, quarantine, runtime-root, relay, SSH, and relay pre-install setup waits attach one reaction to each shared promise and explicitly remove 10,000 cancelled caller closures while one anchor remains. Built-entry macOS harnesses kill the desktop/runtime and SSH relay watcher children, then require automatic resubscription and later events while the host process and relay PTY survive. SSH bundles the child boundary beside relay.js, requires both artifacts for install completeness, and preserves registration-owned same-root cancellation.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/5308", @@ -7008,9 +6598,7 @@ }, { "file": "config/scripts/package-electron-runtime-contract.test.mjs", - "assertions": [ - "every release platform packages and gates the hashed watcher child" - ] + "assertions": ["every release platform packages and gates the hashed watcher child"] }, { "file": "config/scripts/runtime-file-watcher-resource-probe.mjs", @@ -7324,30 +6912,12 @@ "provider terminal inventory", "relay capability enforcement" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime", - "mobile-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime", "mobile-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "ssh"], "coverageNotes": "Deterministic macOS contract evidence covers opaque local- and SSH-shaped terminal ids, one bounded worktree listing, mismatch rejection, and the main/relay host-call adapter matrix. It does not launch a live PTY or provision a relay-hosted plugin.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/8549" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/8549"], "invariant": "terminal.sendText accepts only an explicit provider-owned terminal id present in one bounded inventory of the worktree resolved immediately before the send; an absent id causes zero send calls, and relay callers cannot supply their own capability grants or transport classification.", "oracle": "Resolve the active worktree once, list that worktree with the v0 terminal cap once, and assert zero sendTerminal calls for a mismatched opaque id versus exactly one send for matching local- and SSH-shaped ids; then run the same permission and schema cases through desktop-main and registered relay panel/worker adapters and compare error codes.", "commands": [ diff --git a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md index 80cc68a8468..9193c802648 100644 --- a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md +++ b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md @@ -2,14 +2,16 @@ Date: 2026-07-27 -Status: architecture reviewed; incremental implementation plan ready +Status: architecture reviewed; SSH V1 implementation complete behind an +experimental startup gate; final branch/CI validation pending -Baseline: `origin/main` at `0edc95fa35e4b6e403e116d4e224ad35329b0159` -after rebasing this design on 2026-07-27 +Historical unbounded baseline: `badf91101babf96fa09cb79a8294f7e23b9f081c` +(the implementation branch parent). The final validation record is updated +after rebasing this PR onto current `origin/main`. ## Scope -This plan bounds SSH relay PTY output from the native PTY through the relay +This implemented design bounds SSH relay PTY output from the native PTY through the relay dispatcher, SSH channel or relay socket, Electron main process, and renderer parser. It covers: @@ -25,6 +27,50 @@ work does not change PTY input semantics, terminal-model interpretation, or file/Git payload semantics. It does change replay transport, SSH producer pause wiring, and bulk frame admission where they share the dispatcher sink. +## Implemented architecture and evidence boundary + +The architecture review approved one shared semantic +`PtyConsumerSession` state machine with an SSH-specific transport adapter. +It did not approve a universal wire protocol. In this PR: + +- `PtyConsumerSession` owns authenticated client/owner generations, lease + recovery, capability intersection, and grant publication commit/rollback. +- `SshPtyConsumerSessionAdapter` binds those semantics to the existing + authenticated relay connection and the first framed `pty.openClient` + request. The relay dispatcher remains the only sink-publication authority. +- The relay source ledger owns immutable source spans, outstanding source + credit, token rotation, sealed exit, and cancellation proof. The dispatcher + writer owns byte admission, priorities, `write(false)`, callbacks, and drain. +- `SshRelaySession` owns grant/reconnect state, bounded recovery quarantine, + exact recovery fencing, and cumulative ACK publication through the main SSH + mux. +- The main SSH intake owns one atomic admission across the model, desktop + projection, and required remote consumers. Model completion and consumer + terminality make ACKs eligible; only the mux write callback publishes them. +- Desktop and remote replacement transactions carry immutable span identity. + Reserve happens before publication, commit requires the exact + generation/sequence fence, and failure or stale publication rolls back + without credit. + +The implementation is intentionally scoped: + +| Surface | State in this PR | Evidence | +| ------------------------------------------- | ------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Same-build direct SSH/deployed relay, V1 on | implemented | deterministic relay/main contracts plus macOS-hosted Docker OpenSSH | +| Same-build SSH, V1 off | implemented bounded legacy mode | deterministic rollout and writer contracts | +| Reconnect/owner recovery | implemented | exact contiguous recovery, stale-owner one-shot retry, eight-wide reattach tests, Docker reconnect | +| Headed/headless remote consumers | source-range reserve/commit/rollback implemented | deterministic runtime/main seam only; no live paired-runtime claim | +| WSL stdio and Windows named pipe/ConPTY | shared transport paths preserved | deterministic/common-contract evidence only; no physical run | +| Local provider and local daemon | unchanged | outside SSH V1 negotiation; existing behavior only | +| Folder workspace | no `.git` dependency added | code-path review only; no dedicated live fixture | +| Prior-version/mixed-version peers | fail-closed or version-scoped legacy behavior preserved | deterministic rollout/deploy contracts; no live old binary | +| Ubuntu 20.04/glibc 2.31 | no native dependency added | cross-target relay build only; no physical packaging run | + +Docker SSH proves the Linux SSH provider and deployed-relay topology. It does +not prove headed or headless paired-runtime behavior, WSL, Windows ConPTY or +named pipes, local daemon/provider behavior, folder workspaces, prior-version +processes, mixed-version clients, or the Ubuntu 20.04 packaging floor. + ## Verified baseline and migration boundary The SSH path is not currently protected by the main pending-data bound: @@ -213,6 +259,20 @@ ship independently. A relay-only `pty.getCapabilities` followed by `pty.negotiateClient` is rejected because it would create a second readiness authority beside the existing authenticated handshakes. +The 2026-07-28 architecture/terminal review closed its three blocking +findings as follows: + +| Finding | Implemented resolution | +| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Exited delivery removed while uncredited spans remain | Native exit seals the token; logical state survives until cumulative ACK, exact transfer, token-scoped cancellation proof, or generation close. | +| Terminal obligations depended circularly on ACK publication | Obligation terminality advances `obligationsTerminalEndSu`; the coalescer independently queues `ackQueuedEndSu`; only mux write settlement advances `ackPublishedEndSu`. | +| Desktop projection admission lacked immutable range identity and rollback | Model, source span, projection range, and scanner snapshot reserve atomically; pre-commit failure rolls back, while post-commit replacement transfers with proof. | + +Later adversarial review also required exact recovery continuity, stale-attempt +isolation, startup-latched rollout semantics, and idempotent remote detach +after token reclamation. Those are incorporated in the activation, lifecycle, +cleanup, tests, and rollout sections below. + The common state machine accepts only: - an authenticated principal and owner-eligibility decision from the adapter; @@ -259,20 +319,17 @@ the semantic fields, and calls the fence. The reviewed decision preserves the existing connection machinery: -| Path | Authentication and identity | Shared-session binding | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| Local in-process provider | trusted main-process construction | instantiate semantics directly; no RPC or capability probe | -| Local daemon | token-authenticated `HelloMessage`, stable `clientId`, paired control/stream sockets, and `daemonIdentity` | control hello stores an optional offer; stream hello response publishes the grant after pairing | -| SSH relay socket/named pipe | private endpoint credential validated by `--connect` before its sentinel | first framed `pty.openClient` carries the optional offer; its response is the provider-readiness fence | -| Direct SSH stdio and WSL child stdio | launch-bound ephemeral nonce consumed by the relay adapter | first framed `pty.openClient` proves the nonce and carries the offer; its response is the readiness fence | -| Remote-runtime server | paired-device/E2EE identity and connection ID | adapt `terminal.multiplex` subscriptions; never issue SSH owner leases | +| Path | Authentication and identity | Binding in this PR | +| --------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Local in-process provider | trusted main-process construction | unchanged; it does not negotiate SSH V1 | +| Local daemon | token-authenticated `HelloMessage`, stable `clientId`, paired control/stream sockets, and `daemonIdentity` | unchanged; an exact-version daemon remains outside the SSH wire contract | +| SSH relay socket/named pipe | existing authenticated relay endpoint and dispatcher client identity | first framed `pty.openClient`; response settlement is the readiness fence | +| Primary relay stdio and WSL child stdio | no authenticated dispatcher principal in this PR | bounded legacy transport only; an unproved client cannot become session owner | +| Remote-runtime server | paired-device/E2EE identity and connection ID | source-range reserve/commit/rollback only; never receives an SSH owner lease | -Current direct stdio and WSL have a one-way sentinel, not a bidirectional -pre-dispatch handshake. The reviewed SSH binding therefore chooses one -mandatory post-sentinel `pty.openClient` request for every SSH adapter; there -is no remaining readiness-vs-request choice. After extracting the exact -sentinel and residue, main sends this as the first framed request and prohibits -spawn, attach, or ordinary dispatch until its response validates. The relay +The implemented owner path is the authenticated detached relay endpoint used +by the SSH provider. After extracting the exact sentinel and residue, main +sends `pty.openClient` before spawn, attach, or ordinary dispatch. The relay adapter queues later decoded frames behind the session response's write callback, so an eager next request cannot race activation. Main marks the provider ready only after receiving the valid response. @@ -281,29 +338,24 @@ Detached `--connect` first reads the version-scoped endpoint credential and authenticates its Unix-socket/named-pipe bridge before emitting its sentinel. The adapter converts that proof into a principal and `allowSessionOwner = true`; `pty.openClient` carries no secret to the generic -RPC handler. Direct ssh2, system SSH, and WSL launches receive a safely quoted -one-use nonce from their existing platform-specific command builder. The -pre-dispatch adapter consumes and strips the nonce from `pty.openClient`; a -manual relay without valid proof is subscriber-only. The nonce expires on the -first accepted open or process exit. +RPC handler. The primary relay stdin/stdout dispatcher identity remains +explicitly unproved and therefore subscriber-only; there is no launch nonce in +this implementation. WSL child stdio shares the bounded decoder/writer +transport changes but does not negotiate an SSH V1 owner in this PR. A manual +relay without the version-scoped endpoint credential is also subscriber-only. The review compared sentinel extension and the single request against one readiness authority, no first-spawn race, authenticated owner replacement, response publication fencing, reconnect idempotency, exact residue transfer, -and deterministic testability. The single request wins because it works with -the current one-way sentinel on ssh2, system SSH, and WSL. A separate -capability probe, two-step negotiation, and a generic cross-transport wire API -are not approved. +and deterministic testability. The single request wins for the authenticated +deployed-relay bridge. A separate capability probe, two-step negotiation, and +a generic cross-transport wire API are not approved. -The daemon adapter preserves its current sequential connection order. The -control hello is authenticated and answered immediately with daemon identity; -an optional offer is only pending state keyed by that authenticated -`clientId`. The later stream hello proves the socket pair, applies the common -state transition once, and may return the grant in its response. The client -does not mark the daemon connected until that response. An implementation -changes the daemon protocol version as usual, so an exact-version prior daemon -continues independently rather than partially interpreting the optional -fields. +The local daemon adapter is deliberately not changed in this PR. Its existing +sequential control/stream hello, authentication, exact-version fencing, and +prior-version isolation remain authoritative. Reusing the semantic state +machine there is a later design decision and is not required to ship the SSH +bound. The optional V1 offer is present only when the main rollout gate is on. The server intersects the offer with its supported versions and clamps @@ -338,10 +390,10 @@ transport client and installed generation. `pty.attach` must receive request context just as `pty.spawn` does. Spawn failure, identity mismatch, stale context, and response cancellation create no active V1 subscription. A legacy spawn/attach returns the current identity/replay shape and creates only the -bounded transport subscription above—no token or source coordinate. During -kill-switch rotation, an old V1 token may be closing while its replacement -legacy transport subscription is opening, but no legacy token exists and no -live token changes mode. +bounded transport subscription above—no token or source coordinate. The +startup-latched mode never rotates a live V1 token into legacy service; normal +shutdown closes the old token before a restarted gate-off main opens a bounded +legacy connection. Session ownership is granted by the authenticated session hello, not constructor-assigned: @@ -378,16 +430,16 @@ other than the one installed by its synchronous response hook. The constructor stdout client is never implicitly a session owner. Production launches detached on POSIX and Windows, invalidates stdout, and connects the desktop bridge through `attachClient` over the versioned Unix socket or named pipe. -Direct stdio receives owner authority only from its launch-bound session hello. +Unproved direct stdio remains subscriber-only. Capability support and rollout enablement are separate. Fresh POSIX and Windows launches receive a safely quoted `--pty-output-flow-control=v1|off` argument controlling advertisement. A new main may decline V1 from an already-running detached relay even if it still -advertises it. Disabling the main gate rotates current tokens with the bounded -cancel/reattach procedure below; it never silently changes a live token's -semantics. Sink drain, writer ordering, decoder bounds, and header-ACK -hardening are correctness fixes and are not disabled by this gate. +advertises it. Changing the gate requires a main-process restart and never +changes a live token's semantics. Sink drain, writer ordering, decoder bounds, +and header-ACK hardening are correctness fixes and are not disabled by this +gate. Production deployment does not form arbitrary mixed-build main/relay pairs. `computeRemoteRelayDir` content-hash-scopes the install directory and its @@ -401,7 +453,7 @@ diagnostic `unsupported-version-skew`, not a rollout cohort. The reachable upgrade skew is an orphaned prior-version daemon in its old version directory with live PTYs while the new main connects to a new endpoint. It retains its old behavior until its own grace/cleanup completes and is not -reachable by the new main's kill switch. Upgrade diagnostics enumerate these +reachable by the new main's startup gate. Upgrade diagnostics enumerate these versioned orphan processes; V1 neither adopts their PTYs nor claims to bound their memory. @@ -669,6 +721,25 @@ direct/manual compatibility clients; it is targeted, producer-admitted rather than control-queued, never broadcast, and returns `restoreRequired` instead of writing a body that cannot fit current non-reserved capacity. +Main quarantines the entire candidate recovery transaction until it can prove +the fence. The first source-bearing recovery range must start at the accepted +checkpoint, every later range must be exactly contiguous, and the final end +must equal `recoveryEndSu`. Empty recovery is valid only when the checkpoint +already equals `recoveryEndSu`; main retains that end as the required start of +the first later live frame even after activation. The first live frame and +every successor must also be exactly contiguous. Gap, overlap, incomplete +suffix, missing body, bad empty recovery, or bad live handoff reaches neither +the model nor desktop projection. + +A failed or stale recovery attempt cancels only its own replacement token. +It never sends cancellation on a newer mux, clears a newer checkpoint, shuts +down the physical PTY/process, deletes PTY ownership, or expires the owner +lease. A valid token-scoped cancellation proof is applied locally when it +matches; otherwise the lease remains detached and retryable. A fresh relay +that no longer retains the cached owner returns typed error `-32041`; main +clears only the cached owner/checkpoints and retries `pty.openClient` exactly +once without `resume`. + `pty.serialize` follows the same rule: its response is a bounded stream marker, snapshot chunks use the producer-owned bulk lane, and a completion fence ends the stream. No replay-, gap-, or serialize-bearing response embeds its body in @@ -1231,6 +1302,17 @@ replacement owner and atomically transfer every remaining mapping; otherwise cancel that consumer and its bounded stream. Old stream generations cannot settle new mappings. +The main registry stores the immutable source identity with each admitted +remote mapping, not only its `spanId`. A replacement reservation moves the +named consumer obligations to `transferring`; only an accepted headed or +headless `SnapshotEnd` publication at or beyond the reservation's required +sequence commits them. Stale generation, failed publication, disconnect, or +explicit rollback restores the original live-stream obligation. If +token-scoped cancellation proof has already reclaimed the authoritative span, +late ACK, detach, rollback, and replacement commit prune the cached mapping +and become idempotent no-ops; they never dereference reclaimed state or create +credit. + The stall policy is deliberately split. Desktop-only parse failure reaches its bounded projection cap, transfers to model restore, and lets upstream credit continue. A stalled model receipt or attached lossless remote has no automatic @@ -1256,18 +1338,22 @@ explicit/connection-close proof. The transaction creates the replacement owner before marking source `transferring`; only exact recovery-complete coverage terminally transfers it. -| Event | Mandatory transition | -| ----------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| model accepts; desktop/mobile parses | settle that consumer | -| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker | -| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting | -| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains | -| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof | -| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore | -| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage | -| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range | -| explicit live-token reset/kill switch | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport | -| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof | +| Event | Mandatory transition | +| ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| model accepts; desktop/mobile parses | settle that consumer | +| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker | +| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting | +| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains | +| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof | +| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore | +| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage | +| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range | +| empty or non-empty recovery completion | require checkpoint-to-`recoveryEndSu` continuity, retain that live-start anchor after activation, then admit only an exactly contiguous live frame | +| stale/overlapping recovery attempt | cancel only that attempt's token; never mutate the replacement mux, checkpoint, PTY ownership, physical process, or owner lease | +| remote snapshot replacement | reserve immutable span IDs/ranges, commit only after current-generation `SnapshotEnd` coverage, otherwise roll back | +| late remote detach after token cancellation | prune already-reclaimed mappings and detach idempotently without dereference or new credit | +| explicit live-token reset | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport | +| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof | There is no “abandon live token” transition. A main-side path that cannot prove settlement, transfer, or relay cancellation must keep the ledger or close the @@ -1330,7 +1416,10 @@ main token span: main receive token: unseen --beforeResolve--> receiving-activation receiving-activation --validated response/no recovery--> active - receiving-activation --source-ranged recovery + complete fence--> active + receiving-activation --quarantine exact checkpoint..recoveryEnd + complete fence--> + active(expectedLiveStart = recoveryEnd) + active --first/next live starts at expectedLiveStart--> active(advance expectedLiveStart) + receiving-activation|active --gap/overlap/bad fence--> canceling-only-this-token active --pty.exit--> exit-sealed exit-sealed --prior model/projection/remote terminal--> exit-ack-queued exit-ack-queued --runtime + renderer exit cleanup--> exited-awaiting-ack-publication @@ -1339,6 +1428,12 @@ main receive token: canceling-exited-token --token cancellation proof--> closed(restore-required) canceling-exited-token --proof cannot publish--> closing-provider receiving-activation|active --overflow/cancel/close--> closed + +remote projection replacement: + live --reserve immutable spans + required SnapshotEnd seq--> transferring + transferring --current generation SnapshotEnd covers required seq--> transferred + transferring --stale/failure/disconnect--> live(rollback) + live|transferring --authoritative token already canceled--> detached(prune cached spans) ``` ```ts @@ -1424,6 +1519,25 @@ function onAckWriteSettled(token: DeliveryToken, endSu: number, result: SinkWrit reclaimPublishedPrefix(token) maybeCloseSealedDelivery(token) } + +function finishRecovery(candidate: RecoveryCandidate, fence: RecoveryComplete): void { + const expectedStart = candidate.checkpointSourceEndSu + const recoveredEnd = validateExactContiguousQuarantine(candidate.frames, expectedStart) + if (recoveredEnd !== fence.recoveryEndSu) { + cancelReplacementTokenOnly(candidate.identity, 'recovery-coverage-mismatch') + return + } + commitQuarantinedRecovery(candidate) + installLiveStartAnchor(candidate.identity, fence.recoveryEndSu) +} + +function replaceRemoteConsumer(stream: RemoteStream, requiredSeq: number): void { + const reservation = remoteRanges.reserveImmutableSpans(stream, requiredSeq) + publishSnapshotEnd(stream).then( + (publication) => remoteRanges.commitIfCurrentAndCovered(reservation, publication), + (error) => remoteRanges.rollback(reservation, error) + ) +} ``` Token/span validation, charged admission, and every membership obligation occur @@ -1573,6 +1687,16 @@ source checkpoint is authoritative. Exit before reattach may still synthesize `code: -1` after `notFound`; retaining remote exit tombstones is a separate behavior change and is not required for the memory bound. +Recovery receipt is transactional. Main retains quarantined recovery bodies, +the candidate mux, token, checkpoint, and `recoveryEndSu` until the exact +completion fence validates. It commits no partial body to the model. On +failure it requests cancellation only through the candidate mux and accepts +only matching proof; a stale attempt may not touch current mux state. After a +successful empty recovery it keeps `recoveryEndSu` as a live continuity anchor +until the first live frame arrives. Failure marks the existing PTY lease +detached and retryable rather than deleting ownership or terminating the +physical process. + For an operation-ID spawn replay, physical result lookup completes first; the current request then creates and fences a new subscription as specified above. Disconnect after physical commit/before response leaves the PTY retained but @@ -1727,12 +1851,12 @@ without parsing logs. ## Incremental implementation map -Do not implement this design as one architectural rewrite. Each slice below is -independently reviewable, keeps legacy framing valid, and must land with its -own deterministic oracle. Later slices may revise file placement, but not the -semantic boundaries or invariants. +This PR implemented the design as the independently testable layers below, +without replacing local/daemon protocols or requiring every deployment +topology to enable V1. The slices remain useful review and rollback +boundaries even though they ship in one PR. -### Slice 1: hostile-header and accounting hardening +### Slice 1: hostile-header and accounting hardening — implemented - In `src/main/ssh/ssh-channel-multiplexer.ts`, clamp header ACKs and delete only present timestamp keys; cap retained timestamps. @@ -1740,7 +1864,7 @@ semantic boundaries or invariants. and liveness rebasing. - No relay, daemon, runtime, or provider protocol change. -### Slice 2: bounded decoder turns +### Slice 2: bounded decoder turns — implemented - In both relay protocol decoders, bound frames and time per turn, preserve a synchronous first handshake frame, pause reads during continuation, and @@ -1750,7 +1874,7 @@ semantic boundaries or invariants. - No PTY credit or session negotiation; prior-version peers keep the same frame format. -### Slice 3: one drain-aware legacy writer +### Slice 3: one drain-aware legacy writer — implemented - Add a concretely named dispatcher writer owning ordinary, bulk, control, sentinel, residue, and `--connect` bridge bytes. @@ -1780,7 +1904,7 @@ semantic boundaries or invariants. and detached relay socket/named-pipe clients without changing the wire. The separate local terminal daemon is not exercised or changed. -### Slice 4: bounded main SSH intake +### Slice 4: bounded main SSH intake — implemented - Make `SshRelaySession.wireUpPtyEvents` hand each event exactly once to a main-only intake under `src/main/ipc/`; SSH no longer bypasses the existing @@ -1801,7 +1925,7 @@ semantic boundaries or invariants. - Extend the existing Docker ACK-stall test only for behavior this slice implements: direct SSH/deployed relay/desktop intake and active typing. -### Slice 5a: SSH session semantics +### Slice 5a: SSH session semantics — implemented for SSH - Add the transport-neutral `PtyConsumerSession` state machine and only the SSH readiness adapter decision above. @@ -1811,7 +1935,7 @@ semantic boundaries or invariants. - Local in-process, daemon hello, and remote-runtime adapters reuse semantics only when their own changes need it; they are not prerequisites for SSH V1. -### Slice 5b: direct SSH source credit +### Slice 5b: direct SSH source credit — implemented - Add the relay immutable span ledger/scheduler and tokenized cumulative ACKs for one direct SSH desktop/model consumer path. @@ -1820,7 +1944,7 @@ semantic boundaries or invariants. implementation. - Gate the feature and retain exact same-build legacy behavior. -### Slice 5c: exit, reconnect, and replay +### Slice 5c: exit, reconnect, and replay — implemented - Add sealed-unsettled exit, cancellation proofs, owner reconnect grace, token replacement, and exact transfer. @@ -1830,7 +1954,7 @@ semantic boundaries or invariants. - Do not enable reconnect V1 until late-ACK, timeout, supersession, and generation-close oracles pass. -### Slice 5d: required remote consumers +### Slice 5d: required remote consumers — deterministic seam implemented - Map remote `terminal.multiplex` encoded-byte ACK frames to accepted source ranges without changing its transport authentication or granting it an SSH @@ -1843,17 +1967,17 @@ semantic boundaries or invariants. - Extend the extracted `terminal-output-frame-chunks.ts` seam with immutable composite mapping input while preserving `694363805`'s code-unit scanner, allocation profile, sequence rounding, and equivalence benchmark. -- Add headed paired-server first, then headless `orca serve`; neither is - inferred from Docker SSH. +- Live headed paired-server and headless `orca serve` validation remains + required before claiming those topologies; neither is inferred from Docker + SSH. - Keep best-effort remote streams outside upstream obligations. -### Slice 5e: remaining adapters and rollout +### Slice 5e: remaining adapters and rollout — deliberately deferred -- Adapt local in-process and daemon hello only if shared session semantics - materially reduce their own lifecycle duplication; neither must adopt SSH - framing or source credit for the SSH release. -- Add WSL, Windows ConPTY/named-pipe, local daemon/provider, folder workspace, - mixed-version, and prior-version-orphan evidence separately. +- Local in-process and daemon hello remain unchanged; neither adopts SSH + framing or source credit in this PR. +- WSL, Windows ConPTY/named-pipe, local daemon/provider, folder workspace, + mixed-version, and prior-version-orphan live evidence remains separate. - Promote reliability-gate provider coverage only for topologies with recorded executable evidence. @@ -1863,7 +1987,28 @@ native dependencies, or a per-file lint exception. ## Tests -### Unit and property tests +The checklist below is the normative matrix, not a claim that every physical +topology ran. The executable implementation evidence in this PR is: + +- shared session negotiation, grant publication, stale-owner recovery, source + ledger, ACK queue/publication separation, and cancellation proof contracts; +- relay writer admission/drain/liveness, zero-display source publication, + operation-ID retry activation, fixed-size filesystem compatibility, + capacity-aware Git/PTY slicing, decoder caps, and token rotation; +- main model admission, desktop identity/rollback, exit barriers, remote + reserve/commit/rollback, token cancellation, exact recovery continuity, + eight-wide reconnect, retention budgets, rollout latching, and deploy policy; +- the four-test Docker OpenSSH/deployed-relay suite for source-window plateau, + concurrent typing, fixed-size filesystem/Git churn, and owner reconnect. + +The deterministic recovery seam specifically covers an empty recovery followed +by a gapped live frame, overlapping reconnect attempts, late frames after +cancel, typed stale-owner retry, and the negative assertions that no physical +PTY shutdown, provider-state clear, ownership delete, or lease expiry occurs. +The remote-consumer seam covers token cancellation followed by late detach and +replacement commit, proving reclaimed span IDs are pruned idempotently. + +### Normative unit and property matrix - accept monotonic cumulative ACKs and reject duplicates, regressions, over-credit, invalid numbers, wrong clients, wrong PTYs, and stale tokens; @@ -2025,24 +2170,23 @@ one last chunk to prove the transient overshoot bound. ### Integration and E2E -The existing `tests/e2e/ssh-docker-relay-perf.spec.ts` is baseline evidence, -not V1 evidence. Its ACK-stall case holds desktop renderer ACKs, builds -main-to-renderer pressure, and checks active typing responsiveness through -direct SSH and the deployed relay. It does not saturate relay stdout/socket -writable queues, pause the remote PTY, exercise source-credit ACKs, prove model -admission bounds, or create a paired Orca runtime. +`tests/e2e/ssh-docker-relay-perf.spec.ts` now contains V1 evidence for the +implemented Linux SSH/deployed-relay path. With +`ORCA_SSH_PTY_SOURCE_CREDIT_V1=1`, it: -Add Docker assertions only after the corresponding runtime slice exists: +1. stalls desktop ACK and observes an exact 256 Ki-source-unit negotiated + plateau while a second SSH PTY remains responsive; +2. preserves fixed `STREAM_CHUNK_SIZE` filesystem semantics and completes Git + churn without corruption while active typing stays within budget; +3. reconnects the negotiated owner lease and proves the existing SSH + workspace terminal remains usable; +4. retains the direct typing latency control. -1. Slice 3 saturates actual direct-SSH/deployed-relay stdio or socket output, - proves `write(false)` stops later ordinary frames, writable length - plateaus, control progresses, and interactive echo stays bounded. -2. Slice 4 stalls main model admission and desktop ACKs independently, proves - charged intake plateaus, and preserves the current active-typing oracle. -3. Slices 5b/5c exhaust the negotiated source window, exercise sealed-exit late - ACK and cancellation, reconnects a new owner generation, rejects stale - ACKs, and verifies exact model/renderer output. -4. A separate concurrent bulk case proves the writer reserve and completion. +Writer `write(false)`/drain, callback failure, liveness bypass, decoder +scheduling, sealed exit, stale ACK, exact recovery, and remote replacement are +proved at deterministic unit/service seams. The Docker suite does not force +the main-to-relay sink to remain saturated beyond the health deadline, does +not prove paired-runtime behavior, and must not be cited for those claims. Docker SSH proves the Linux SSH provider/relay path only. It must never be reported as headed paired-server, headless `orca serve`, remote-runtime @@ -2068,21 +2212,22 @@ adopt their PTYs, or claim the new main bounds their memory. Keep any new native artifact compatible with Ubuntu 20.04/glibc 2.31; this design itself adds none. -The current -`terminal-performance.output-backpressure-budget` reliability gate is -experimental and partial. It lists local, daemon, SSH, and remote-runtime -providers, but `coveredProviders` is empty and its executable evidence is -local macOS main-to-renderer/runtime behavior. Slice 3 adds relay -write/drain evidence, Slice 4 adds SSH intake/model admission, Slices 5b/5c add -source-credit lifecycle, and Slices 5d/5e add topology-specific runs before any -provider is marked covered. This documentation PR does not change or promote -that gate and does not claim unimplemented runtime behavior was exercised. +The +`terminal-performance.output-backpressure-budget` reliability gate remains +`experimental` and `partial`. Its `coveredProviders: ["ssh"]` means the +deterministic direct-SSH contracts plus the macOS-hosted Docker +OpenSSH/deployed-relay run described above. It does not cover every SSH host +platform and does not mark local, daemon, remote-runtime, headed, headless, +WSL, Windows, folder-workspace, prior-version, mixed-version, or Ubuntu 20.04 +packaging topologies as executed. ## Rollout -Land V1 behind a main runtime gate and a relay advertisement launch policy. -Tests and development enable both. Main never negotiates merely because a -long-lived relay advertises; its gate is the final authority. Fresh POSIX and +V1 is behind a main-process startup gate and a persisted per-endpoint relay +launch policy. `ORCA_SSH_PTY_SOURCE_CREDIT_V1` is read once when the main +process starts; it is not a live kill switch. Tests and development enable the +gate before startup. Main never negotiates merely because a long-lived relay +advertises; its startup-latched gate is the final authority. Fresh POSIX and Windows launch commands carry the explicit advertisement argument rather than assuming a local environment variable reaches the remote process. @@ -2092,15 +2237,16 @@ independently. Same-build gate-off sessions use the bounded legacy writer but do not wait for V1 credit. A prior-version orphan remains governed by its own binary and cleanup. -Rollout stages: +Implementation and rollout stages: -1. land Slices 1-4 independently under legacy framing, with each narrow - reliability oracle green before the next slice; -2. add Slices 5a-5c behind the main gate and relay advertisement, initially in - unit/service contracts and direct Docker SSH only; -3. add Slices 5d/5e for headed paired, headless, WSL, Windows, local-daemon, - local-provider, and folder-workspace evidence before marking those - providers/topologies covered; +1. Slices 1-4 are implemented under legacy-compatible framing with narrow + reliability oracles. +2. Slices 5a-5c are implemented behind the startup gate and relay launch + policy with unit/service contracts and direct Docker SSH evidence. +3. Slice 5d's remote source-range transaction is implemented at deterministic + main/runtime seams. Live headed paired, headless, WSL, Windows, + local-daemon, local-provider, and folder-workspace evidence remains future + work before those topologies are marked covered. 4. run an internal same-build canary comparing gate-off and V1 sink, model-admission, latency, sealed-exit, recovery, and reconnect metrics while separately counting orphan prior-version daemons; @@ -2109,27 +2255,25 @@ Rollout stages: fallback and versioned-orphan cleanup stay healthy through the supported upgrade window. -Kill-switch behavior is explicit: +Rollback behavior is explicit: -1. Disable the main gate to stop all new V1 negotiations against the - same-build detached relay. -2. For each live token, stop new desktop membership changes, flush eligible - cumulative ACKs, and retain - `obligationsTerminalEndSu`/`ackQueuedEndSu`/`ackPublishedEndSu` until their - callbacks settle. Send `pty.cancelDelivery` for every remaining active or - sealed-unsettled token and wait up to 10 seconds for cancellation proof. -3. Reattach the same PTYs in bounded waves without requesting V1. Failure to - prove cancellation closes/reconnects the client so relay generation cleanup - is the proof; local ledgers and desktop range identities are not discarded - earlier. -4. Changing the relay launch policy affects fresh processes. Replacing a - long-lived remote relay is required only to remove its implementation or - advertisement, not to make a gated main decline the feature. +1. Change the environment/launch setting, then restart the main process. The + existing process keeps its original mode until shutdown. +2. Shutdown performs the normal token-scoped cancellation/exit cleanup; it + does not rotate live V1 tokens into legacy mode in place. +3. On the next connection, gate-off omits the V1 offer and accepts same-build + bounded legacy service. Gate-on requires a V1-launched endpoint. +4. If a persisted detached endpoint was launched gate-off, an off-to-on + connection fails with reset/reconnect guidance rather than silently + downgrading. Resetting that endpoint and reconnecting launches it with the + new policy. +5. A gate-off client remains compatible with a same-build V1-capable relay + because omission of the offer selects bounded legacy behavior. -Rollback never mutates a live token into legacy mode. The remote process may -remain alive throughout a successful cancel/reattach rotation. An orphaned -prior-version daemon is outside this sequence and is only observed until its -own version-scoped cleanup. +The remote process may remain alive only when its persisted launch policy is +compatible with the new startup mode. An orphaned prior-version daemon is +outside this sequence and is only observed until its own version-scoped +cleanup. Release criteria: @@ -2165,7 +2309,8 @@ Release criteria: truncation. - Content-hashed endpoints make mixed production binaries unreachable. The real upgrade skew is an orphaned prior-version daemon outside the new main's - kill switch; V1 reports but cannot adopt, cancel, or retroactively bound it. + startup-gated endpoint; V1 reports but cannot adopt, cancel, or retroactively + bound it. - Preserving the real exit code for a PTY that exits before any new client reattaches would require attach-visible tombstone retention. V1 documents and tests the existing synthetic `-1` fallback instead of coupling that separate diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index daf15c65a03..ec25c7df6a5 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -159,7 +159,11 @@ import { } from './pty-hidden-delivery-gate' import { PtyPendingDataDrainQueue, type PendingPtyData } from './pty-pending-data-drain-queue' import { SshPtyOutputIntake } from './ssh-pty-output-intake' -import { installSshPtyOutputIntake, publishSshPtySourceAck } from './ssh-pty-output-intake-registry' +import { + cancelSshPtySourceDelivery, + installSshPtyOutputIntake, + publishSshPtySourceAck +} from './ssh-pty-output-intake-registry' import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' import { clearNativeWindowsConptyPty, @@ -3169,7 +3173,8 @@ export function registerPtyHandlers( } } }, - publishSourceAck: publishSshPtySourceAck + publishSourceAck: publishSshPtySourceAck, + cancelSourceDelivery: cancelSshPtySourceDelivery }) runtime?.setRemoteTerminalSourceRangeConsumerHooks?.( sshOutputIntake.getRemoteSourceRangeConsumerHooks() diff --git a/src/main/ipc/ssh-pty-model-admission-pressure.ts b/src/main/ipc/ssh-pty-model-admission-pressure.ts new file mode 100644 index 00000000000..18837e6c51c --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-pressure.ts @@ -0,0 +1,145 @@ +import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract' +import { + admissionKeyId, + pressureHasAdmissionKey, + takePausedGeneration, + type AdmissionEntry, + type PtyUsage +} from './ssh-pty-model-admission-entry' +import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits' + +type PressureOptions = { + limits: SshPtyModelAdmissionLimits + pauseProvider: (key: SshPtyModelAdmissionKey) => boolean + resumeProvider: (key: SshPtyModelAdmissionKey) => void +} + +type PromotionOptions = { + usageByPty: ReadonlyMap + disposed: boolean + canReserve: (entry: AdmissionEntry) => boolean + reserve: (entry: AdmissionEntry) => void + isBelowGlobalLowWatermark: () => boolean +} + +export class SshPtyModelAdmissionPressure { + private readonly entries: AdmissionEntry[] = [] + private readonly pausedKeys = new Map() + private retainedBytes = 0 + + constructor(private readonly options: PressureOptions) {} + + get values(): readonly AdmissionEntry[] { + return this.entries + } + + get frameCount(): number { + return this.entries.length + } + + get bytes(): number { + return this.retainedBytes + } + + get pausedPtyCount(): number { + return this.pausedKeys.size + } + + has(key: SshPtyModelAdmissionKey): boolean { + return pressureHasAdmissionKey(this.entries, key) + } + + admit(entry: AdmissionEntry): boolean { + const id = admissionKeyId(entry.key) + const paused = this.pausedKeys.has(id) || this.options.pauseProvider(entry.key) + if (paused) { + this.pausedKeys.set(id, entry.key) + } + if ( + !paused || + this.entries.length >= this.options.limits.pressureMaxFrames || + this.retainedBytes + entry.charge.bytes > this.options.limits.pressureMaxBytes + ) { + return false + } + entry.state = 'pressure' + this.entries.push(entry) + this.retainedBytes += entry.charge.bytes + return true + } + + cancelGeneration( + providerGeneration: number, + cancelPressureAndReserved: (entries: AdmissionEntry[]) => number + ): void { + this.retainedBytes -= cancelPressureAndReserved(this.entries) + for (const key of takePausedGeneration(this.pausedKeys, providerGeneration)) { + this.resume(key) + } + } + + cancelPty(key: SshPtyModelAdmissionKey, error: Error, cancelReserved: () => void): void { + const id = admissionKeyId(key) + const canceled: AdmissionEntry[] = [] + for (let index = this.entries.length - 1; index >= 0; index--) { + if (admissionKeyId(this.entries[index]!.key) === id) { + canceled.push(this.entries.splice(index, 1)[0]!) + } + } + this.rejectPressureEntries(canceled, error) + cancelReserved() + const paused = this.pausedKeys.get(id) + if (paused) { + this.pausedKeys.delete(id) + this.resume(paused) + } + } + + promoteAndResume(options: PromotionOptions): void { + if (options.disposed) { + return + } + for (let index = 0; index < this.entries.length; ) { + const entry = this.entries[index]! + const hasEarlierEntryForPty = this.entries + .slice(0, index) + .some((earlier) => admissionKeyId(earlier.key) === admissionKeyId(entry.key)) + if (hasEarlierEntryForPty || !options.canReserve(entry)) { + index++ + continue + } + this.entries.splice(index, 1) + this.retainedBytes -= entry.charge.bytes + options.reserve(entry) + } + if (!options.isBelowGlobalLowWatermark()) { + return + } + for (const [id, key] of this.pausedKeys) { + const usage = options.usageByPty.get(id) + if ( + this.has(key) || + (usage?.sourceUnits ?? 0) > this.options.limits.perPtyLowSourceUnits || + (usage?.bytes ?? 0) > this.options.limits.perPtyLowBytes + ) { + continue + } + this.pausedKeys.delete(id) + this.resume(key) + } + } + + private rejectPressureEntries(entries: readonly AdmissionEntry[], error: Error): void { + for (const entry of entries) { + this.retainedBytes -= entry.charge.bytes + entry.state = 'settled' + entry.reject(error) + } + } + + private resume(key: SshPtyModelAdmissionKey): void { + try { + this.options.resumeProvider(key) + } catch {} + } +} diff --git a/src/main/ipc/ssh-pty-model-admission.ts b/src/main/ipc/ssh-pty-model-admission.ts index e074b707aed..1f387ddd516 100644 --- a/src/main/ipc/ssh-pty-model-admission.ts +++ b/src/main/ipc/ssh-pty-model-admission.ts @@ -9,10 +9,8 @@ import { admissionKeyId, canReserveAdmission, cancelAdmissionGeneration, - pressureHasAdmissionKey, retainedBytes, resolveAdmissionIdleWaiters, - takePausedGeneration, type AdmissionCharge, type AdmissionEntry, type PtyUsage @@ -21,6 +19,7 @@ import { resolveSshPtyModelAdmissionLimits, type SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits' +import { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure' export type { SshPtyModelAdmissionKey, @@ -30,23 +29,22 @@ export type { export class SshPtyModelAdmission { private readonly limits: SshPtyModelAdmissionLimits - private readonly pauseProvider: (key: SshPtyModelAdmissionKey) => boolean - private readonly resumeProvider: (key: SshPtyModelAdmissionKey) => void private readonly closeProvider: (providerGeneration: number, reason: string) => void private readonly usageByPty = new Map() - private readonly pressure: AdmissionEntry[] = [] - private readonly pausedKeys = new Map() + private readonly pressure: SshPtyModelAdmissionPressure private readonly idleWaiters = new Map void>>() private readonly closingGenerations = new Set() private globalSourceUnits = 0 private globalBytes = 0 - private pressureBytes = 0 private disposed = false constructor(options: SshPtyModelAdmissionOptions = {}) { this.limits = resolveSshPtyModelAdmissionLimits(options) - this.pauseProvider = options.pauseProvider ?? (() => false) - this.resumeProvider = options.resumeProvider ?? (() => {}) + this.pressure = new SshPtyModelAdmissionPressure({ + limits: this.limits, + pauseProvider: options.pauseProvider ?? (() => false), + resumeProvider: options.resumeProvider ?? (() => {}) + }) this.closeProvider = options.closeProvider ?? (() => {}) } @@ -72,45 +70,65 @@ export class SshPtyModelAdmission { reject, state: 'queued' } - if (this.canReserve(key, charge) && !pressureHasAdmissionKey(this.pressure, key)) { + if (this.canReserve(key, charge) && !this.pressure.has(key)) { this.reserveAndQueue(entry) return } - this.admitPressure(entry) + if (!this.pressure.admit(entry)) { + entry.reject(admissionError('ssh_model_admission_pressure_exhausted')) + this.closeProvider(entry.key.providerGeneration, 'model-admission-pressure') + } }) } closeGeneration(providerGeneration: number, reason = 'provider-generation-closed'): void { this.closingGenerations.add(providerGeneration) const error = admissionError(reason) - this.pressureBytes -= cancelAdmissionGeneration({ - pressure: this.pressure, - usageByPty: this.usageByPty, - idleWaiters: this.idleWaiters, - providerGeneration, - error, - release: (key, charge) => this.release(key, charge) - }) - for (const key of takePausedGeneration(this.pausedKeys, providerGeneration)) { - try { - this.resumeProvider(key) - } catch {} - } + this.pressure.cancelGeneration(providerGeneration, (pressure) => + cancelAdmissionGeneration({ + pressure, + usageByPty: this.usageByPty, + idleWaiters: this.idleWaiters, + providerGeneration, + error, + release: (key, charge) => this.release(key, charge) + }) + ) const generationPrefix = `${providerGeneration}\0` for (const id of this.idleWaiters.keys()) { if (id.startsWith(generationPrefix)) { - resolveAdmissionIdleWaiters(this.usageByPty, this.pressure, this.idleWaiters, id) + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) } } } + cancelPty(key: SshPtyModelAdmissionKey, reason: string): void { + const id = admissionKeyId(key) + const error = admissionError(reason) + this.pressure.cancelPty(key, error, () => { + const usage = this.usageByPty.get(id) + if (usage) { + const canceled = [...usage.queued, ...(usage.running ? [usage.running] : [])] + usage.queued = [] + usage.running = null + for (const entry of canceled) { + if (entry.state === 'settled') { + continue + } + entry.state = 'settled' + this.release(entry.key, entry.charge) + entry.reject(error) + } + this.usageByPty.delete(id) + } + }) + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) + } + whenIdle(key: SshPtyModelAdmissionKey): Promise { const id = admissionKeyId(key) const usage = this.usageByPty.get(id) - const hasPressure = this.pressure.some( - (entry) => - entry.key.providerGeneration === key.providerGeneration && entry.key.ptyId === key.ptyId - ) + const hasPressure = this.pressure.has(key) if ((!usage || (!usage.running && usage.queued.length === 0)) && !hasPressure) { return Promise.resolve() } @@ -135,7 +153,7 @@ export class SshPtyModelAdmission { generations.add(entry.key.providerGeneration) } } - for (const entry of this.pressure) { + for (const entry of this.pressure.values) { generations.add(entry.key.providerGeneration) } for (const generation of generations) { @@ -147,9 +165,9 @@ export class SshPtyModelAdmission { return { sourceUnits: this.globalSourceUnits, bytes: this.globalBytes, - pressureFrames: this.pressure.length, - pressureBytes: this.pressureBytes, - pausedPtys: this.pausedKeys.size + pressureFrames: this.pressure.frameCount, + pressureBytes: this.pressure.bytes, + pausedPtys: this.pressure.pausedPtyCount } } @@ -181,33 +199,13 @@ export class SshPtyModelAdmission { this.startNext(id, usage) } - private admitPressure(entry: AdmissionEntry): void { - const id = admissionKeyId(entry.key) - const paused = this.pausedKeys.has(id) || this.pauseProvider(entry.key) - if (paused) { - this.pausedKeys.set(id, entry.key) - } - if ( - !paused || - this.pressure.length >= this.limits.pressureMaxFrames || - this.pressureBytes + entry.charge.bytes > this.limits.pressureMaxBytes - ) { - entry.reject(admissionError('ssh_model_admission_pressure_exhausted')) - this.closeProvider(entry.key.providerGeneration, 'model-admission-pressure') - return - } - entry.state = 'pressure' - this.pressure.push(entry) - this.pressureBytes += entry.charge.bytes - } - private startNext(id: string, usage: PtyUsage): void { if (usage.running) { return } const entry = usage.queued.shift() if (!entry) { - this.maybeResumeAndPromote() + this.promotePressureAndResumeProvider() return } usage.running = entry @@ -233,13 +231,24 @@ export class SshPtyModelAdmission { }) }, (error) => { - if (this.finishEntry(id, usage, entry)) { - entry.reject(error instanceof Error ? error : new Error(String(error))) - } + this.failEntry(id, usage, entry, error instanceof Error ? error : new Error(String(error))) } ) } + private failEntry(id: string, usage: PtyUsage, entry: AdmissionEntry, error: Error): void { + if (entry.state !== 'running' || usage.running !== entry) { + return + } + this.closingGenerations.add(entry.key.providerGeneration) + usage.running = null + entry.state = 'settled' + this.release(entry.key, entry.charge) + entry.reject(error) + this.closeGeneration(entry.key.providerGeneration, 'ssh_model_admission_completion_failed') + this.cleanupUsage(id, usage) + } + private finishEntry(id: string, usage: PtyUsage, entry: AdmissionEntry): boolean { if (entry.state !== 'running' || usage.running !== entry) { return false @@ -256,7 +265,7 @@ export class SshPtyModelAdmission { if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) { this.usageByPty.delete(id) } - resolveAdmissionIdleWaiters(this.usageByPty, this.pressure, this.idleWaiters, id) + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) } private release(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): void { @@ -267,49 +276,18 @@ export class SshPtyModelAdmission { } this.globalSourceUnits = Math.max(0, this.globalSourceUnits - charge.sourceUnits) this.globalBytes = Math.max(0, this.globalBytes - charge.bytes) - this.maybeResumeAndPromote() + this.promotePressureAndResumeProvider() } - private maybeResumeAndPromote(): void { - if (this.disposed) { - return - } - for (let index = 0; index < this.pressure.length; ) { - const entry = this.pressure[index]! - const hasEarlierEntryForPty = this.pressure - .slice(0, index) - .some((earlier) => admissionKeyId(earlier.key) === admissionKeyId(entry.key)) - if (hasEarlierEntryForPty || !this.canReserve(entry.key, entry.charge)) { - index++ - continue - } - this.pressure.splice(index, 1) - this.pressureBytes -= entry.charge.bytes - this.reserveAndQueue(entry) - } - if ( - this.globalSourceUnits > this.limits.globalLowSourceUnits || - this.globalBytes > this.limits.globalLowBytes - ) { - return - } - for (const [id, key] of this.pausedKeys) { - const usage = this.usageByPty.get(id) - const stillPressured = this.pressure.some( - (entry) => - entry.key.providerGeneration === key.providerGeneration && entry.key.ptyId === key.ptyId - ) - if ( - stillPressured || - (usage?.sourceUnits ?? 0) > this.limits.perPtyLowSourceUnits || - (usage?.bytes ?? 0) > this.limits.perPtyLowBytes - ) { - continue - } - this.pausedKeys.delete(id) - try { - this.resumeProvider(key) - } catch {} - } + private promotePressureAndResumeProvider(): void { + this.pressure.promoteAndResume({ + usageByPty: this.usageByPty, + disposed: this.disposed, + canReserve: (entry) => this.canReserve(entry.key, entry.charge), + reserve: (entry) => this.reserveAndQueue(entry), + isBelowGlobalLowWatermark: () => + this.globalSourceUnits <= this.limits.globalLowSourceUnits && + this.globalBytes <= this.limits.globalLowBytes + }) } } diff --git a/src/main/ipc/ssh-pty-output-exit-deadline.ts b/src/main/ipc/ssh-pty-output-exit-deadline.ts new file mode 100644 index 00000000000..2dbe3cd3343 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-exit-deadline.ts @@ -0,0 +1,161 @@ +import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection' +import type { SshPtyModelAdmission } from './ssh-pty-model-admission' +import type { + SshPtyOutputExitEvent, + SshPtyOutputIntakeDependencies +} from './ssh-pty-output-intake-contract' +import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation' +import type { SshPtyOutputSourceObligations } from './ssh-pty-output-source-obligations' + +type SshPtyOutputExitDeadlineDependencies = Readonly<{ + admission: SshPtyModelAdmission + projections: SshPtyLegacyProjectionLedger + sourceObligations: SshPtyOutputSourceObligations + intake: SshPtyOutputIntakeDependencies + barrierMs?: number + cancellationProofMs?: number +}> + +export class SshPtyOutputExitDeadline { + private readonly barriersByGeneration = new Map>() + private readonly barrierMs: number + private readonly cancellationProofMs: number + + constructor(private readonly dependencies: SshPtyOutputExitDeadlineDependencies) { + this.barrierMs = dependencies.barrierMs ?? 30_000 + this.cancellationProofMs = dependencies.cancellationProofMs ?? 10_000 + } + + wait(event: SshPtyOutputExitEvent, promise: Promise): Promise { + return new Promise((resolve, reject) => { + let timeoutStarted = false + let settled = false + const settle = (result: { ok: true } | { ok: false; error: Error }): void => { + if (settled) { + return + } + settled = true + clearTimeout(barrier.timer) + this.remove(event.providerGeneration, barrier) + if (result.ok) { + resolve() + } else { + reject(result.error) + } + } + const barrier: SshPtyExitBarrier = { + timer: setTimeout(() => { + timeoutStarted = true + void this.cancelTimedOutExit(event).then( + () => settle({ ok: true }), + (error) => { + this.dependencies.intake.closeProvider?.( + event.providerGeneration, + 'ssh-exit-cancellation-proof-failed' + ) + settle({ + ok: false, + error: error instanceof Error ? error : outputIntakeError(String(error)) + }) + } + ) + }, this.barrierMs), + reject: (error) => settle({ ok: false, error }) + } + barrier.timer.unref?.() + let barriers = this.barriersByGeneration.get(event.providerGeneration) + if (!barriers) { + barriers = new Set() + this.barriersByGeneration.set(event.providerGeneration, barriers) + } + barriers.add(barrier) + void promise.then( + () => { + if (!timeoutStarted) { + settle({ ok: true }) + } + }, + (error) => { + if (!timeoutStarted) { + settle({ ok: false, error }) + } + } + ) + }) + } + + closeGeneration(providerGeneration: number, error: Error): void { + const barriers = this.barriersByGeneration.get(providerGeneration) + if (!barriers) { + return + } + for (const barrier of barriers) { + clearTimeout(barrier.timer) + barrier.reject(error) + } + this.barriersByGeneration.delete(providerGeneration) + } + + get activeBarriers(): number { + return Array.from(this.barriersByGeneration.values()).reduce( + (total, barriers) => total + barriers.size, + 0 + ) + } + + private async cancelTimedOutExit(event: SshPtyOutputExitEvent): Promise { + const cancel = this.dependencies.intake.cancelSourceDelivery + if (!cancel) { + throw outputIntakeError('ssh_source_cancellation_publisher_unavailable') + } + this.dependencies.admission.cancelPty( + { ptyId: event.id, providerGeneration: event.providerGeneration }, + 'ssh_exit_delivery_canceled' + ) + this.dependencies.sourceObligations.sealPty(event) + const cancellation = this.dependencies.sourceObligations.cancelPty(event, (request) => + cancel(event.providerGeneration, request) + ) + const canceled = await this.withCancellationProofDeadline(cancellation) + if (!canceled) { + throw outputIntakeError('ssh_source_cancellation_identity_unavailable') + } + this.dependencies.projections.transferPty(event.id, 'ssh-exit-delivery-canceled') + this.dependencies.intake.prepareExit(event) + this.dependencies.intake.finalizeExit(event) + this.dependencies.projections.closePty( + event.id, + event.providerGeneration, + event.ptyIncarnation, + 'ssh-exit-delivery-canceled' + ) + } + + private withCancellationProofDeadline(promise: Promise): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(outputIntakeError('ssh_source_cancellation_proof_timeout')), + this.cancellationProofMs + ) + timer.unref?.() + void promise.then( + (value) => { + clearTimeout(timer) + resolve(value) + }, + (error) => { + clearTimeout(timer) + reject(error) + } + ) + }) + } + + private remove(providerGeneration: number, barrier: SshPtyExitBarrier): void { + const barriers = this.barriersByGeneration.get(providerGeneration) + barriers?.delete(barrier) + if (barriers?.size === 0) { + this.barriersByGeneration.delete(providerGeneration) + } + } +} diff --git a/src/main/ipc/ssh-pty-output-exit.ts b/src/main/ipc/ssh-pty-output-exit.ts index 03e9f8e4c9a..2828679f3dd 100644 --- a/src/main/ipc/ssh-pty-output-exit.ts +++ b/src/main/ipc/ssh-pty-output-exit.ts @@ -11,14 +11,26 @@ export async function settleSshPtyOutputExit(args: { projections: SshPtyLegacyProjectionLedger dependencies: SshPtyOutputIntakeDependencies validateGeneration: () => void + afterAdmissionIdle?: () => void + waitForSourceTerminal?: () => Promise beforeFinalize?: () => void }): Promise { - const { event, admission, projections, dependencies, validateGeneration, beforeFinalize } = args + const { + event, + admission, + projections, + dependencies, + validateGeneration, + afterAdmissionIdle, + waitForSourceTerminal, + beforeFinalize + } = args await admission.whenIdle({ ptyId: event.id, providerGeneration: event.providerGeneration }) validateGeneration() + afterAdmissionIdle?.() try { dependencies.prepareExit(event) } catch (error) { @@ -38,6 +50,7 @@ export async function settleSshPtyOutputExit(args: { 'pty-exit-unpublished' ) await projections.whenPtyTerminal(event.id, event.providerGeneration, event.ptyIncarnation) + await waitForSourceTerminal?.() validateGeneration() try { beforeFinalize?.() diff --git a/src/main/ipc/ssh-pty-output-intake-contract.ts b/src/main/ipc/ssh-pty-output-intake-contract.ts index de54429c381..50776c67778 100644 --- a/src/main/ipc/ssh-pty-output-intake-contract.ts +++ b/src/main/ipc/ssh-pty-output-intake-contract.ts @@ -5,6 +5,18 @@ import type { } from './ssh-pty-model-admission-contract' import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract' +export type SshPtySourceCancellationRequest = Readonly<{ + id: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string +}> + +export type SshPtySourceCancellationProof = Readonly<{ + sentEndSu: number + creditedEndSu: number +}> + export type SshPtyOutputDataEvent = Readonly<{ id: string data: string @@ -52,8 +64,13 @@ export type SshPtyOutputIntakeDependencies = { batch: PtySourceCreditAckBatch, onSettled: (result: { ok: true } | { ok: false; error: Error }) => void ) => void + cancelSourceDelivery?: ( + providerGeneration: number, + request: SshPtySourceCancellationRequest + ) => Promise } export type SshPtyOutputIntakeOptions = SshPtyModelAdmissionOptions & { exitBarrierMs?: number + exitCancellationProofMs?: number } diff --git a/src/main/ipc/ssh-pty-output-intake-registry.ts b/src/main/ipc/ssh-pty-output-intake-registry.ts index 103171f9557..655ecf8154e 100644 --- a/src/main/ipc/ssh-pty-output-intake-registry.ts +++ b/src/main/ipc/ssh-pty-output-intake-registry.ts @@ -2,7 +2,9 @@ import type { SshPtyOutputIntake } from './ssh-pty-output-intake' import type { SshPtyOutputDataEvent, SshPtyOutputExitEvent, - SshPtyOutputReceipt + SshPtyOutputReceipt, + SshPtySourceCancellationProof, + SshPtySourceCancellationRequest } from './ssh-pty-output-intake-contract' import type { SshPtyAcceptedSourceCheckpoint } from './ssh-pty-output-source-obligations' import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract' @@ -10,12 +12,17 @@ import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-con let installedIntake: SshPtyOutputIntake | null = null let nextProviderGeneration = 1 const sourceAckPublishers = new Map() +const sourceCancellationPublishers = new Map() type SshPtySourceAckPublisher = ( batch: PtySourceCreditAckBatch, onSettled: (result: { ok: true } | { ok: false; error: Error }) => void ) => void +type SshPtySourceCancellationPublisher = ( + request: SshPtySourceCancellationRequest +) => Promise + export function allocateSshPtyProviderGeneration(): number { return nextProviderGeneration++ } @@ -82,6 +89,38 @@ export function publishSshPtySourceAck( publisher(batch, onSettled) } +export function installSshPtySourceCancellationPublisher( + providerGeneration: number, + cancel: SshPtySourceCancellationPublisher +): () => void { + if (sourceCancellationPublishers.has(providerGeneration)) { + throw new Error('ssh_source_cancellation_publisher_duplicate_generation') + } + sourceCancellationPublishers.set(providerGeneration, cancel) + return () => { + if (sourceCancellationPublishers.get(providerGeneration) === cancel) { + sourceCancellationPublishers.delete(providerGeneration) + } + } +} + +export function cancelSshPtySourceDelivery( + providerGeneration: number, + request: SshPtySourceCancellationRequest +): Promise { + const publisher = sourceCancellationPublishers.get(providerGeneration) + return publisher + ? publisher(request) + : Promise.reject(new Error('ssh_source_cancellation_publisher_unavailable')) +} + +export function applySshPtySourceCancellationProof( + event: SshPtyOutputExitEvent, + proof: SshPtySourceCancellationProof +): boolean { + return installedIntake?.applySourceCancellationProof(event, proof) ?? false +} + function outputIntakeUnavailableError(): Error { return Object.assign(new Error('ssh_output_intake_unavailable'), { code: 'ssh_output_intake_unavailable' diff --git a/src/main/ipc/ssh-pty-output-intake.test.ts b/src/main/ipc/ssh-pty-output-intake.test.ts index 370894a34aa..332da53ab58 100644 --- a/src/main/ipc/ssh-pty-output-intake.test.ts +++ b/src/main/ipc/ssh-pty-output-intake.test.ts @@ -376,6 +376,65 @@ describe('SshPtyOutputIntake', () => { expect(harness.order).toEqual(['model:aaaa', 'project:aaaa', 'exit']) }) + it('admits queued pre-exit source spans before sealing the token', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const first = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + const second = harness.intake.acceptData( + event({ + data: 'bbbb', + source: { + spanId: 'span-2', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 4, + sourceEndSu: 8 + } + }) + ) + const exit = harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + + harness.completions[0]!.resolve() + await first + harness.completions[1]!.resolve() + await Promise.all([second, exit]) + expect(harness.order).toEqual([ + 'model:aaaa', + 'project:aaaa', + 'model:bbbb', + 'project:bbbb', + 'exit' + ]) + }) + + it('does not pump queued model work after a running completion fails', async () => { + const harness = createHarness() + const first = harness.intake.acceptData(event({ data: 'first' })) + const queued = harness.intake.acceptData(event({ data: 'queued' })) + + harness.completions[0]!.reject(new Error('emulator failed')) + await expect(first).rejects.toThrow('emulator failed') + await expect(queued).rejects.toThrow('ssh_model_admission_completion_failed') + expect(harness.order).toEqual(['model:first', 'project:first']) + expect(harness.completions).toHaveLength(1) + }) + it('retains exited delivery until published renderer projections settle', async () => { const harness = createHarness({}, { exitBarrierMs: 1000 }) const dataReceipt = harness.intake.acceptData(event()) @@ -408,8 +467,57 @@ describe('SshPtyOutputIntake', () => { expect(harness.intake.getDebugSnapshot().projection.records).toBe(0) }) + it('retains exit until a required remote source consumer settles', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const remote = harness.intake.getRemoteSourceRangeConsumerHooks() + const stream = { + ptyId: 'pty-1', + consumerId: 'remote-1', + streamGeneration: 'stream-1' + } + expect(remote.attach(stream)).toBe(true) + const dataReceipt = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + const receipt = await dataReceipt + + let exited = false + const exitReceipt = harness.intake + .acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + .then(() => { + exited = true + }) + await Promise.resolve() + expect(exited).toBe(false) + + remote.settle(stream, [receipt.projection.desktopSpan!]) + await exitReceipt + expect(harness.order.at(-1)).toBe('exit') + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + }) + it('keeps timed-out exit projections until generation-close proof', async () => { - const harness = createHarness({}, { exitBarrierMs: 1 }) + const harness = createHarness( + { + cancelSourceDelivery: () => Promise.reject(new Error('cancel transport failed')) + }, + { exitBarrierMs: 1, exitCancellationProofMs: 10 } + ) const dataReceipt = harness.intake.acceptData(event()) harness.completions[0]!.resolve() const receipt = await dataReceipt @@ -426,14 +534,67 @@ describe('SshPtyOutputIntake', () => { providerGeneration: 1, ptyIncarnation: 'incarnation-1' }) - ).rejects.toThrow('ssh_exit_barrier_timeout') - expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'ssh-exit-barrier-timeout') + ).rejects.toThrow('ssh_source_cancellation_identity_unavailable') + expect(harness.dependencies.closeProvider).toHaveBeenCalledWith( + 1, + 'ssh-exit-cancellation-proof-failed' + ) expect(harness.intake.getDebugSnapshot().projection.records).toBe(1) harness.intake.closeGeneration(1, 'provider-closed') expect(harness.intake.getDebugSnapshot().projection.records).toBe(0) }) + it('cancels only the timed-out source delivery and keeps the provider usable', async () => { + const cancelSourceDelivery = vi.fn(async () => ({ sentEndSu: 4, creditedEndSu: 0 })) + const harness = createHarness( + { cancelSourceDelivery }, + { exitBarrierMs: 1, exitCancellationProofMs: 100 } + ) + const remote = harness.intake.getRemoteSourceRangeConsumerHooks() + const stream = { + ptyId: 'pty-1', + consumerId: 'remote-1', + streamGeneration: 'stream-1' + } + remote.attach(stream) + const dataReceipt = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + await dataReceipt + + await harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + expect(cancelSourceDelivery).toHaveBeenCalledWith(1, { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect(harness.order.at(-1)).toBe('exit') + + const sibling = harness.intake.acceptData( + event({ id: 'pty-2', ptyIncarnation: 'incarnation-2' }) + ) + harness.completions[1]!.resolve() + await expect(sibling).resolves.toMatchObject({ ptyId: 'pty-2' }) + }) + it('rejects late same-generation data after ordered exit cleanup', async () => { const harness = createHarness({}, { exitBarrierMs: 1000 }) const first = harness.intake.acceptData(event()) diff --git a/src/main/ipc/ssh-pty-output-intake.ts b/src/main/ipc/ssh-pty-output-intake.ts index 4cbaddb7088..a95101dca84 100644 --- a/src/main/ipc/ssh-pty-output-intake.ts +++ b/src/main/ipc/ssh-pty-output-intake.ts @@ -3,6 +3,7 @@ import { type LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' import { SshPtyModelAdmission } from './ssh-pty-model-admission' +import { SshPtyOutputExitDeadline } from './ssh-pty-output-exit-deadline' import { settleSshPtyOutputExit } from './ssh-pty-output-exit' import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard' import { @@ -16,7 +17,7 @@ import type { SshPtyOutputIntakeOptions, SshPtyOutputReceipt } from './ssh-pty-output-intake-contract' -import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation' +import { outputIntakeError } from './ssh-pty-output-intake-validation' export type { SshPtyOutputDataEvent, @@ -31,8 +32,7 @@ export class SshPtyOutputIntake { private readonly sourceObligations: SshPtyOutputSourceObligations private readonly generationGuard = new SshPtyOutputGenerationGuard(() => this.disposed) private readonly admission: SshPtyModelAdmission - private readonly exitBarrierMs: number - private readonly exitBarriersByGeneration = new Map>() + private readonly exitDeadline: SshPtyOutputExitDeadline private disposed = false constructor( @@ -44,7 +44,6 @@ export class SshPtyOutputIntake { onSettled: (span) => this.sourceObligations.settleDesktop(span, 'renderer-parse'), onTransferred: (span, reason) => this.sourceObligations.transferDesktop(span, reason) }) - this.exitBarrierMs = options.exitBarrierMs ?? 30_000 this.admission = new SshPtyModelAdmission({ ...options, pauseProvider: (key) => @@ -54,6 +53,14 @@ export class SshPtyOutputIntake { closeProvider: (providerGeneration, reason) => this.dependencies.closeProvider?.(providerGeneration, reason) }) + this.exitDeadline = new SshPtyOutputExitDeadline({ + admission: this.admission, + projections: this.projections, + sourceObligations: this.sourceObligations, + intake: this.dependencies, + barrierMs: options.exitBarrierMs, + cancellationProofMs: options.exitCancellationProofMs + }) } acceptData(event: SshPtyOutputDataEvent): Promise { @@ -141,7 +148,9 @@ export class SshPtyOutputIntake { 'model-admission-failed' ) } - this.dependencies.closeProvider?.(event.providerGeneration, 'model-admission-failed') + if ((error as { code?: unknown }).code !== 'ssh_exit_delivery_canceled') { + this.dependencies.closeProvider?.(event.providerGeneration, 'model-admission-failed') + } throw error } ) @@ -149,8 +158,7 @@ export class SshPtyOutputIntake { async acceptExit(event: SshPtyOutputExitEvent): Promise { this.generationGuard.sealExit(event) - this.sourceObligations.sealPty(event) - await this.withExitDeadline(event.providerGeneration, this.finishExit(event)) + await this.exitDeadline.wait(event, this.finishExit(event)) } private async finishExit(event: SshPtyOutputExitEvent): Promise { @@ -160,6 +168,8 @@ export class SshPtyOutputIntake { projections: this.projections, dependencies: this.dependencies, validateGeneration: () => this.generationGuard.validate(event), + afterAdmissionIdle: () => this.sourceObligations.sealPty(event), + waitForSourceTerminal: () => this.sourceObligations.whenPtyTerminal(event), beforeFinalize: () => this.sourceObligations.markExitPublished(event) }) } @@ -196,14 +206,7 @@ export class SshPtyOutputIntake { this.dependencies.onGenerationClosed?.(providerGeneration, reason) this.projections.closeGeneration(providerGeneration, reason) this.sourceObligations.closeGeneration(providerGeneration, reason) - const barriers = this.exitBarriersByGeneration.get(providerGeneration) - if (barriers) { - for (const barrier of barriers) { - clearTimeout(barrier.timer) - barrier.reject(outputIntakeError(reason)) - } - this.exitBarriersByGeneration.delete(providerGeneration) - } + this.exitDeadline.closeGeneration(providerGeneration, outputIntakeError(reason)) } dispose(): void { @@ -226,55 +229,19 @@ export class SshPtyOutputIntake { return this.sourceObligations.acceptedCheckpoints(providerGeneration) } + applySourceCancellationProof( + event: SshPtyOutputExitEvent, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): boolean { + return this.sourceObligations.applyCancellationProof(event, proof) + } + getDebugSnapshot() { return { model: this.admission.getDebugSnapshot(), projection: this.projections.getDebugSnapshot(), source: this.sourceObligations.getDebugSnapshot(), - exitBarriers: Array.from(this.exitBarriersByGeneration.values()).reduce( - (total, barriers) => total + barriers.size, - 0 - ) - } - } - - private withExitDeadline(providerGeneration: number, promise: Promise): Promise { - return new Promise((resolve, reject) => { - const barrier: SshPtyExitBarrier = { - timer: setTimeout(() => { - this.removeExitBarrier(providerGeneration, barrier) - this.dependencies.closeProvider?.(providerGeneration, 'ssh-exit-barrier-timeout') - reject(outputIntakeError('ssh_exit_barrier_timeout')) - }, this.exitBarrierMs), - reject - } - barrier.timer.unref?.() - let barriers = this.exitBarriersByGeneration.get(providerGeneration) - if (!barriers) { - barriers = new Set() - this.exitBarriersByGeneration.set(providerGeneration, barriers) - } - barriers.add(barrier) - void promise.then( - () => { - clearTimeout(barrier.timer) - this.removeExitBarrier(providerGeneration, barrier) - resolve() - }, - (error) => { - clearTimeout(barrier.timer) - this.removeExitBarrier(providerGeneration, barrier) - reject(error) - } - ) - }) - } - - private removeExitBarrier(providerGeneration: number, barrier: SshPtyExitBarrier): void { - const barriers = this.exitBarriersByGeneration.get(providerGeneration) - barriers?.delete(barrier) - if (barriers?.size === 0) { - this.exitBarriersByGeneration.delete(providerGeneration) + exitBarriers: this.exitDeadline.activeBarriers } } } diff --git a/src/main/ipc/ssh-pty-output-source-obligations.ts b/src/main/ipc/ssh-pty-output-source-obligations.ts index 804cc7d0fb6..9d0a39a9e56 100644 --- a/src/main/ipc/ssh-pty-output-source-obligations.ts +++ b/src/main/ipc/ssh-pty-output-source-obligations.ts @@ -9,7 +9,9 @@ import type { DesktopProjectionSpan } from './ssh-pty-legacy-projection' import type { SshPtyOutputDataEvent, SshPtyOutputExitEvent, - SshPtyOutputIntakeDependencies + SshPtyOutputIntakeDependencies, + SshPtySourceCancellationProof, + SshPtySourceCancellationRequest } from './ssh-pty-output-intake-contract' import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers' import type { SshPtySourceAdmissionReservation } from './ssh-pty-source-obligation-contract' @@ -133,8 +135,39 @@ export class SshPtyOutputSourceObligations { } } + whenPtyTerminal(event: SshPtyOutputExitEvent): Promise { + const identity = this.identityByPty.get(this.ptyKey(event)) + return identity ? this.coordinator.whenTerminal(identity) : Promise.resolve() + } + + async cancelPty( + event: SshPtyOutputExitEvent, + cancel: (request: SshPtySourceCancellationRequest) => Promise + ): Promise { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (!identity) { + return false + } + const request = this.coordinator.beginExitTimeout(identity) + const proof = await cancel(request) + this.coordinator.applyCancellationProof(identity, proof) + return true + } + + applyCancellationProof( + event: SshPtyOutputExitEvent, + proof: SshPtySourceCancellationProof + ): boolean { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (!identity) { + return false + } + this.coordinator.applyCancellationProof(identity, proof) + return true + } + closeGeneration(providerGeneration: number, reason: string): void { - this.remoteConsumers.closeGeneration(providerGeneration) + this.remoteConsumers.closeGeneration(providerGeneration, reason) this.coordinator.closeGeneration(providerGeneration, reason) const prefix = `${providerGeneration}\0` for (const key of this.openedTokens) { diff --git a/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts b/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts index dcf20e6b850..0b34feb9a2d 100644 --- a/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts +++ b/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts @@ -83,7 +83,40 @@ describe('SshPtyRemoteSourceRangeConsumers', () => { expect(progress).toHaveBeenCalledTimes(1) }) - it('atomically transfers remaining mappings on explicit detach', () => { + it.each(['headless', 'renderer'] as const)( + 'commits remaining mappings only after a %s snapshot publication', + (source) => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) + + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + expect(replacement).not.toBeNull() + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'transferring', + to: 'remote:snapshot:consumer-1' + }) + expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 3 })).toBe(false) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('transferring') + expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 4 })).toBe(true) + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'transferred', + to: 'remote:snapshot:consumer-1' + }) + expect(consumers.requiredConsumers('pty-1')).toEqual(['remote:consumer-1']) + } + ) + + it('rolls a failed replacement publication back to the live stream obligation', () => { const ledger = createCoordinator() const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } @@ -96,13 +129,12 @@ describe('SshPtyRemoteSourceRangeConsumers', () => { ledger.commit(reservation) consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) - consumers.hooks.transfer(stream, [range('span-1')], 'stream-detached') + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + expect(consumers.hooks.rollbackReplacement(replacement!, 'snapshot-write-failed')).toBe(true) expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ - state: 'transferred', - to: 'remote:snapshot:consumer-1' + state: 'open' }) - expect(consumers.requiredConsumers('pty-1')).toEqual([]) }) it('settles a split source span only after its complete ordered range is acknowledged', () => { @@ -139,7 +171,7 @@ describe('SshPtyRemoteSourceRangeConsumers', () => { expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled') }) - it('transfers an admitted span even when no encoded mapping was sent', () => { + it('cancels an admitted span on detach without minting a snapshot owner', () => { const ledger = createCoordinator() const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } @@ -152,10 +184,127 @@ describe('SshPtyRemoteSourceRangeConsumers', () => { ledger.commit(reservation) consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) - consumers.hooks.transfer(stream, [], 'stream-detached') + consumers.hooks.cancel(stream, [], 'stream-detached') expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ - state: 'transferred' + state: 'canceled', + reason: 'stream-detached' + }) + expect(consumers.requiredConsumers('pty-1')).toEqual([]) + }) + + it('rolls back a pending replacement before disconnect cancellation', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + consumers.hooks.cancel(stream, [], 'connection-closed') + + expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe( + false + ) + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'canceled', + reason: 'connection-closed' }) }) + + it('rejects stale stream generations without changing the current obligation', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) + + expect(() => + consumers.hooks.reserveReplacement( + { ...stream, streamGeneration: 'stale' }, + 4, + 'initial-snapshot' + ) + ).toThrow('stale') + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + }) + + it('rolls back replacement admission before provider-generation close', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + consumers.closeGeneration(identity.providerGeneration, 'provider-replaced') + + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe( + false + ) + }) + + it('detaches cleanly after cancellation proof reclaims tracked spans', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) + ledger.seal(identity) + ledger.beginExitTimeout(identity) + ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 }) + + expect(() => consumers.hooks.settle(stream, [range('span-1')])).not.toThrow() + expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow() + expect(consumers.requiredConsumers('pty-1')).toEqual([]) + }) + + it('rejects a replacement commit after cancellation proof reclaims its spans', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers) + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + ledger.seal(identity) + ledger.beginExitTimeout(identity) + ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 }) + + expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe( + false + ) + expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow() + expect(consumers.requiredConsumers('pty-1')).toEqual([]) + }) }) diff --git a/src/main/ipc/ssh-pty-remote-source-range-consumers.ts b/src/main/ipc/ssh-pty-remote-source-range-consumers.ts index 80942dcc767..35ccd25d2a8 100644 --- a/src/main/ipc/ssh-pty-remote-source-range-consumers.ts +++ b/src/main/ipc/ssh-pty-remote-source-range-consumers.ts @@ -6,8 +6,10 @@ import type { RemoteTerminalSourceRangeConsumerHooks, RemoteTerminalSourceRangeStreamIdentity } from '../runtime/remote-terminal-source-range-consumer' +import type { PtySourceSpan } from '../../shared/pty-source-credit-contract' import type { SshPtySourceConsumerId } from './ssh-pty-source-obligation-contract' import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' +import { SshPtyRemoteSourceRangeReplacements } from './ssh-pty-remote-source-range-replacement' function remoteConsumerId( identity: RemoteTerminalSourceRangeStreamIdentity @@ -15,34 +17,35 @@ function remoteConsumerId( return `remote:${identity.consumerId}` } -function replacementConsumerId( - identity: RemoteTerminalSourceRangeStreamIdentity -): SshPtySourceConsumerId { - return `remote:snapshot:${identity.consumerId}` -} - function uniqueSpanIds(ranges: readonly TerminalOutputSourceRange[]): string[] { return Array.from(new Set(ranges.map((range) => range.spanId))) } type RemoteConsumerState = { streamGeneration: string - spanGenerations: Map + spans: Map ackedEndBySpan: Map } export class SshPtyRemoteSourceRangeConsumers { private readonly consumersByPty = new Map>() + private readonly replacements: SshPtyRemoteSourceRangeReplacements constructor( private readonly coordinator: SshPtySourceObligationCoordinator, private readonly onProgress: (range: TerminalOutputSourceRange) => void = () => {} - ) {} + ) { + this.replacements = new SshPtyRemoteSourceRangeReplacements(coordinator) + } readonly hooks: RemoteTerminalSourceRangeConsumerHooks = { attach: (identity) => this.attach(identity), settle: (identity, ranges) => this.settle(identity, ranges), - transfer: (identity, ranges, reason) => this.transfer(identity, ranges, reason), + reserveReplacement: (identity, requiredSeq, reason) => + this.reserveReplacement(identity, requiredSeq, reason), + commitReplacement: (reservation, publication) => + this.commitReplacement(reservation, publication), + rollbackReplacement: (reservation, reason) => this.rollbackReplacement(reservation, reason), cancel: (identity, ranges, reason) => this.cancel(identity, ranges, reason) } @@ -61,17 +64,18 @@ export class SshPtyRemoteSourceRangeConsumers { ): void { for (const [consumerId, state] of this.consumersByPty.get(ptyId) ?? []) { if (requiredConsumers.includes(`remote:${consumerId}`)) { - state.spanGenerations.set(spanId, this.coordinator.spanIdentity(spanId).providerGeneration) + state.spans.set(spanId, this.coordinator.spanIdentity(spanId)) } } } - closeGeneration(providerGeneration: number): void { + closeGeneration(providerGeneration: number, reason: string): void { + this.replacements.closeGeneration(providerGeneration, reason) for (const consumers of this.consumersByPty.values()) { for (const state of consumers.values()) { - for (const [spanId, generation] of state.spanGenerations) { - if (generation === providerGeneration) { - state.spanGenerations.delete(spanId) + for (const [spanId, source] of state.spans) { + if (source.providerGeneration === providerGeneration) { + state.spans.delete(spanId) state.ackedEndBySpan.delete(spanId) } } @@ -90,7 +94,7 @@ export class SshPtyRemoteSourceRangeConsumers { identity.consumerId, current ?? { streamGeneration: identity.streamGeneration, - spanGenerations: new Map(), + spans: new Map(), ackedEndBySpan: new Map() } ) @@ -110,10 +114,15 @@ export class SshPtyRemoteSourceRangeConsumers { const nextEnds = new Map(state.ackedEndBySpan) const completed = new Set() for (const range of ranges) { - if (!state.spanGenerations.has(range.spanId)) { + const source = state.spans.get(range.spanId) + if (!source) { + continue + } + if (!this.coordinator.hasRetainedSpan(range.spanId)) { + state.spans.delete(range.spanId) + nextEnds.delete(range.spanId) continue } - const source = this.coordinator.spanIdentity(range.spanId) const currentEnd = nextEnds.get(range.spanId) ?? source.sourceStartSu if ( !sameTerminalOutputSourceIdentity(source, range) || @@ -129,14 +138,19 @@ export class SshPtyRemoteSourceRangeConsumers { } state.ackedEndBySpan = nextEnds for (const spanId of completed) { - const source = this.coordinator.spanIdentity(spanId) + const source = state.spans.get(spanId) + if (!source || !this.coordinator.hasRetainedSpan(spanId)) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + continue + } this.coordinator.settle({ identity: source, spanId, consumer, reason: 'remote-frame-ack' }) - state.spanGenerations.delete(spanId) + state.spans.delete(spanId) state.ackedEndBySpan.delete(spanId) } for (const range of ranges) { @@ -144,40 +158,48 @@ export class SshPtyRemoteSourceRangeConsumers { } } - private transfer( + private reserveReplacement( identity: RemoteTerminalSourceRangeStreamIdentity, - ranges: readonly TerminalOutputSourceRange[], + requiredSeq: number, reason: string - ): void { + ): ReturnType { if (!this.isCurrent(identity)) { throw new Error('ssh_remote_source_range_stale_generation') } - const consumer = remoteConsumerId(identity) - const replacement = replacementConsumerId(identity) const state = this.requireState(identity) - const spanIds = Array.from( - new Set([ - ...state.spanGenerations.keys(), - ...uniqueSpanIds(ranges).filter((spanId) => state.spanGenerations.has(spanId)) - ]) - ) - for (const spanId of spanIds) { - if (this.coordinator.obligation(spanId, consumer).state !== 'open') { - throw new Error('ssh_remote_source_range_transfer_invalid') + for (const spanId of state.spans.keys()) { + if (!this.coordinator.hasRetainedSpan(spanId)) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) } } - for (const spanId of spanIds) { - const source = this.coordinator.spanIdentity(spanId) - this.coordinator.beginTransfer({ identity: source, spanId, consumer, reason }, replacement) - } - for (const spanId of spanIds) { - const source = this.coordinator.spanIdentity(spanId) - this.coordinator.commitTransfer({ identity: source, spanId, consumer }) - } - this.detachIdentity(identity) - for (const range of ranges) { - this.onProgress(range) - } + const spanIds = Array.from(state.spans.keys()) + return this.replacements.reserve(identity, spanIds, requiredSeq, reason) + } + + private commitReplacement( + reservation: Parameters[0], + publication: Parameters[1] + ): boolean { + return this.replacements.commit( + reservation, + publication, + this.isCurrent(reservation.identity), + (spanIds) => { + const state = this.requireState(reservation.identity) + for (const spanId of spanIds) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + } + } + ) + } + + private rollbackReplacement( + reservation: Parameters[0], + reason: string + ): boolean { + return this.replacements.rollback(reservation, reason) } private cancel( @@ -188,14 +210,18 @@ export class SshPtyRemoteSourceRangeConsumers { if (!this.isCurrent(identity)) { return } + this.replacements.rollbackIdentity(identity, `${reason}-replacement-aborted`) const consumer = remoteConsumerId(identity) const state = this.requireState(identity) const spanIds = new Set([ - ...state.spanGenerations.keys(), - ...uniqueSpanIds(ranges).filter((spanId) => state.spanGenerations.has(spanId)) + ...state.spans.keys(), + ...uniqueSpanIds(ranges).filter((spanId) => state.spans.has(spanId)) ]) for (const spanId of spanIds) { - const source = this.coordinator.spanIdentity(spanId) + const source = state.spans.get(spanId) + if (!source || !this.coordinator.hasRetainedSpan(spanId)) { + continue + } const transition = { identity: source, spanId, consumer, reason } if (this.coordinator.beginTransfer(transition, consumer)) { this.coordinator.cancelTransfer(transition) diff --git a/src/main/ipc/ssh-pty-remote-source-range-replacement.ts b/src/main/ipc/ssh-pty-remote-source-range-replacement.ts new file mode 100644 index 00000000000..bf11b34b15c --- /dev/null +++ b/src/main/ipc/ssh-pty-remote-source-range-replacement.ts @@ -0,0 +1,155 @@ +import type { + RemoteTerminalSourceRangeReplacementPublication, + RemoteTerminalSourceRangeReplacementReservation, + RemoteTerminalSourceRangeStreamIdentity +} from '../runtime/remote-terminal-source-range-consumer' +import type { PtySourceSpan } from '../../shared/pty-source-credit-contract' +import type { SshPtySourceConsumerId } from './ssh-pty-source-obligation-contract' +import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' + +type ReplacementReservationRecord = { + reservation: RemoteTerminalSourceRangeReplacementReservation + spans: readonly PtySourceSpan[] + consumer: SshPtySourceConsumerId + replacement: SshPtySourceConsumerId + reason: string +} + +function remoteConsumerId( + identity: RemoteTerminalSourceRangeStreamIdentity +): SshPtySourceConsumerId { + return `remote:${identity.consumerId}` +} + +export class SshPtyRemoteSourceRangeReplacements { + private readonly reservations = new Map() + private nextReservationId = 1 + + constructor(private readonly coordinator: SshPtySourceObligationCoordinator) {} + + reserve( + identity: RemoteTerminalSourceRangeStreamIdentity, + spanIds: readonly string[], + requiredSeq: number, + reason: string + ): RemoteTerminalSourceRangeReplacementReservation | null { + if (spanIds.length === 0) { + return null + } + if (!Number.isSafeInteger(requiredSeq) || requiredSeq < 0) { + throw new Error('ssh_remote_source_range_replacement_sequence_invalid') + } + const consumer = remoteConsumerId(identity) + const replacement = `remote:snapshot:${identity.consumerId}` as const + const spans = spanIds.map((spanId) => this.coordinator.spanIdentity(spanId)) + for (const { spanId } of spans) { + if (this.coordinator.obligation(spanId, consumer).state !== 'open') { + throw new Error('ssh_remote_source_range_transfer_invalid') + } + } + for (const source of spans) { + const { spanId } = source + this.coordinator.beginTransfer({ identity: source, spanId, consumer, reason }, replacement) + } + const reservation = Object.freeze({ + reservationId: `remote-source-replacement:${this.nextReservationId++}`, + identity: Object.freeze({ ...identity }), + requiredSeq + }) + this.reservations.set(reservation.reservationId, { + reservation, + spans: Object.freeze(spans), + consumer, + replacement, + reason + }) + return reservation + } + + commit( + reservation: RemoteTerminalSourceRangeReplacementReservation, + publication: RemoteTerminalSourceRangeReplacementPublication, + isCurrent: boolean, + onCommitted: (spanIds: readonly string[]) => void + ): boolean { + const record = this.reservations.get(reservation.reservationId) + if ( + !record || + record.reservation !== reservation || + !isCurrent || + !Number.isSafeInteger(publication.seq) || + publication.seq < reservation.requiredSeq || + (publication.source !== 'headless' && publication.source !== 'renderer') + ) { + return false + } + if ( + record.spans.some(({ spanId }) => { + if (!this.coordinator.hasRetainedSpan(spanId)) { + return true + } + const obligation = this.coordinator.obligation(spanId, record.consumer) + return obligation.state !== 'transferring' || obligation.to !== record.replacement + }) + ) { + this.reservations.delete(reservation.reservationId) + return false + } + for (const source of record.spans) { + const { spanId } = source + if ( + !this.coordinator.commitTransfer({ identity: source, spanId, consumer: record.consumer }) + ) { + throw new Error('ssh_remote_source_range_replacement_commit_invalid') + } + } + this.reservations.delete(reservation.reservationId) + onCommitted(record.spans.map(({ spanId }) => spanId)) + return true + } + + rollback(reservation: RemoteTerminalSourceRangeReplacementReservation, reason: string): boolean { + const record = this.reservations.get(reservation.reservationId) + if (!record || record.reservation !== reservation) { + return false + } + this.reservations.delete(reservation.reservationId) + let rolledBack = true + for (const source of record.spans) { + const { spanId } = source + if (!this.coordinator.hasRetainedSpan(spanId)) { + continue + } + rolledBack = + this.coordinator.rollbackTransfer({ + identity: source, + spanId, + consumer: record.consumer, + reason + }) && rolledBack + } + return rolledBack + } + + rollbackIdentity(identity: RemoteTerminalSourceRangeStreamIdentity, reason: string): void { + for (const record of Array.from(this.reservations.values())) { + if ( + record.reservation.identity.ptyId === identity.ptyId && + record.reservation.identity.consumerId === identity.consumerId && + record.reservation.identity.streamGeneration === identity.streamGeneration + ) { + this.rollback(record.reservation, reason) + } + } + } + + closeGeneration(providerGeneration: number, reason: string): void { + for (const record of Array.from(this.reservations.values())) { + if ( + record.spans.some(({ providerGeneration: generation }) => generation === providerGeneration) + ) { + this.rollback(record.reservation, `${reason}-replacement-aborted`) + } + } + } +} diff --git a/src/main/ipc/ssh-pty-source-obligation-coordinator.ts b/src/main/ipc/ssh-pty-source-obligation-coordinator.ts index 9d9d0ef4222..e06133abb80 100644 --- a/src/main/ipc/ssh-pty-source-obligation-coordinator.ts +++ b/src/main/ipc/ssh-pty-source-obligation-coordinator.ts @@ -1,4 +1,5 @@ import { + ptySourceDeliveryKey, samePtySourceDelivery, type PtySourceDeliveryIdentity, type PtySourceSpan @@ -22,9 +23,20 @@ export type SshPtySourceObligationTransition = Readonly<{ reason: string }> +type TerminalWaiter = { + resolve: () => void + reject: (error: Error) => void +} + +type TerminalWaiterGroup = { + identity: PtySourceDeliveryIdentity + waiters: Set +} + export class SshPtySourceObligationCoordinator { private readonly ledger: SshPtySourceObligationLedger private readonly acknowledgements: SshPtySourceAckCoalescer + private readonly terminalWaiters = new Map() private disposed = false constructor(options: SshPtySourceAckCoalescerOptions) { @@ -56,7 +68,11 @@ export class SshPtySourceObligationCoordinator { } rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean { - return this.ledger.rollbackCommitted(reservation) + const rolledBack = this.ledger.rollbackCommitted(reservation) + if (rolledBack) { + this.maybeResolveTerminal(reservation.span) + } + return rolledBack } settle(transition: SshPtySourceObligationTransition): boolean { @@ -89,7 +105,15 @@ export class SshPtySourceObligationCoordinator { return changed } + rollbackTransfer(transition: SshPtySourceObligationTransition): boolean { + this.requireSpanIdentity(transition) + return this.ledger.rollbackTransfer(transition.spanId, transition.consumer) + } + seal(identity: PtySourceDeliveryIdentity): void { + if (this.ledger.snapshot(identity).state === 'sealed-unsettled') { + return + } this.ledger.seal(identity) } @@ -98,6 +122,21 @@ export class SshPtySourceObligationCoordinator { this.ledger.markExitPublished(identity) } + whenTerminal(identity: PtySourceDeliveryIdentity): Promise { + if (this.isTerminal(identity)) { + return Promise.resolve() + } + const key = ptySourceDeliveryKey(identity) + let group = this.terminalWaiters.get(key) + if (!group) { + group = { identity: Object.freeze({ ...identity }), waiters: new Set() } + this.terminalWaiters.set(key, group) + } + return new Promise((resolve, reject) => { + group!.waiters.add({ resolve, reject }) + }) + } + beginExitTimeout(identity: PtySourceDeliveryIdentity) { return this.ledger.beginExitTimeout(identity) } @@ -106,10 +145,26 @@ export class SshPtySourceObligationCoordinator { identity: PtySourceDeliveryIdentity, proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> ): void { + const snapshot = this.ledger.snapshot(identity) + if ( + snapshot.state === 'closed' && + snapshot.receivedEndSu === proof.sentEndSu && + snapshot.ackPublishedEndSu === proof.creditedEndSu + ) { + return + } this.ledger.applyCancellationProof(identity, proof) + this.rejectWaiters( + (group) => samePtySourceDelivery(group.identity, identity), + new Error('ssh_source_delivery_canceled') + ) } closeGeneration(providerGeneration: number, reason: string): number { + this.rejectWaiters( + (group) => group.identity.providerGeneration === providerGeneration, + new Error(reason) + ) const closed = this.ledger.closeGeneration(providerGeneration, reason) this.acknowledgements.cancelGeneration(providerGeneration, reason) return closed @@ -131,6 +186,10 @@ export class SshPtySourceObligationCoordinator { return this.ledger.spanIdentity(spanId) } + hasRetainedSpan(spanId: string): boolean { + return this.ledger.hasRetainedSpan(spanId) + } + flushAcknowledgements(): void { this.acknowledgements.flush() } @@ -140,6 +199,7 @@ export class SshPtySourceObligationCoordinator { return } this.disposed = true + this.rejectWaiters(() => true, new Error(reason ?? 'SSH PTY source obligations disposed')) this.ledger.closeAll(reason ?? 'SSH PTY source obligation coordinator disposed') this.acknowledgements.dispose(reason) } @@ -149,6 +209,41 @@ export class SshPtySourceObligationCoordinator { if (publication) { this.acknowledgements.enqueue(publication) } + this.maybeResolveTerminal(identity) + } + + private isTerminal(identity: PtySourceDeliveryIdentity): boolean { + const snapshot = this.ledger.snapshot(identity) + return ( + snapshot.obligationsTerminalEndSu === snapshot.receivedEndSu && + snapshot.ackQueuedEndSu === snapshot.receivedEndSu + ) + } + + private maybeResolveTerminal(identity: PtySourceDeliveryIdentity): void { + if (!this.isTerminal(identity)) { + return + } + const group = this.terminalWaiters.get(ptySourceDeliveryKey(identity)) + if (!group) { + return + } + this.terminalWaiters.delete(ptySourceDeliveryKey(identity)) + for (const waiter of group.waiters) { + waiter.resolve() + } + } + + private rejectWaiters(predicate: (group: TerminalWaiterGroup) => boolean, error: Error): void { + for (const [key, group] of this.terminalWaiters) { + if (!predicate(group)) { + continue + } + this.terminalWaiters.delete(key) + for (const waiter of group.waiters) { + waiter.reject(error) + } + } } private requireSpanIdentity( diff --git a/src/main/ipc/ssh-pty-source-obligation-ledger.ts b/src/main/ipc/ssh-pty-source-obligation-ledger.ts index f45e6296c51..40d81d9af99 100644 --- a/src/main/ipc/ssh-pty-source-obligation-ledger.ts +++ b/src/main/ipc/ssh-pty-source-obligation-ledger.ts @@ -37,6 +37,7 @@ import { cancelSourceObligationTransfer, commitSourceObligationTransfer, modelAcceptedSourceEnd, + rollbackSourceObligationTransfer, transitionOpenSourceObligation } from './ssh-pty-source-obligation-transitions' @@ -160,6 +161,10 @@ export class SshPtySourceObligationLedger { return cancelSourceObligationTransfer(this.spanOwners, spanId, consumer, reason) } + rollbackTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean { + return rollbackSourceObligationTransfer(this.spanOwners, spanId, consumer) + } + queueAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null { const token = this.requireToken(identity) if (token.obligationsTerminalEndSu <= token.ackQueuedEndSu) { @@ -263,6 +268,10 @@ export class SshPtySourceObligationLedger { return requireSourceSpan(this.spanOwners, spanId).span.span } + hasRetainedSpan(spanId: string): boolean { + return this.spanOwners.has(spanId) + } + private transitionOpen( spanId: string, consumer: SshPtySourceConsumerId, diff --git a/src/main/ipc/ssh-pty-source-obligation-transitions.ts b/src/main/ipc/ssh-pty-source-obligation-transitions.ts index f136540cbb4..1008af7879f 100644 --- a/src/main/ipc/ssh-pty-source-obligation-transitions.ts +++ b/src/main/ipc/ssh-pty-source-obligation-transitions.ts @@ -56,6 +56,19 @@ export function cancelSourceObligationTransfer( return true } +export function rollbackSourceObligationTransfer( + spanOwners: ReadonlyMap, + spanId: string, + consumer: SshPtySourceConsumerId +): boolean { + const { span } = requireSourceSpan(spanOwners, spanId) + if (span.obligations.get(consumer)?.state !== 'transferring') { + return false + } + span.obligations.set(consumer, Object.freeze({ state: 'open' })) + return true +} + export function modelAcceptedSourceEnd(token: TokenRecord): number { let acceptedEndSu = token.ackPublishedEndSu for (const record of token.spans) { diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 99bb651ed99..3b55dfc0055 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -684,10 +684,14 @@ describe('SshPtyProvider', () => { replay: 'restored output', incarnationId: 'incarnation-reconnect' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { - id: 'pty-1', - suppressReplayNotification: true - }) + expect(mux.request).toHaveBeenCalledWith( + 'pty.attach', + { + id: 'pty-1', + suppressReplayNotification: true + }, + { timeoutMs: 10_000 } + ) }) it('keeps missing incarnation compatible with an old relay', async () => { @@ -712,12 +716,16 @@ describe('SshPtyProvider', () => { tabId: 'tab-a' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { - id: 'pty-1', - suppressReplayNotification: true, - expectedPaneKey: 'tab-a:leaf-a', - expectedTabId: 'tab-a' - }) + expect(mux.request).toHaveBeenCalledWith( + 'pty.attach', + { + id: 'pty-1', + suppressReplayNotification: true, + expectedPaneKey: 'tab-a:leaf-a', + expectedTabId: 'tab-a' + }, + { timeoutMs: 10_000 } + ) }) it('write sends pty.data notification', () => { diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index a7845bd0a78..1b72ded7d85 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -198,14 +198,15 @@ export class SshPtyProvider implements IPtyProvider { // be filtered before they reach the renderer. The expected identity lets the // relay reject a cross-generation id collision instead of reattaching this // lease to a different pane's freshly spawned PTY. + const params = { + id: this.toRelayPtyId(id), + suppressReplayNotification: true, + ...(sourceRecovery ? { sourceRecovery } : {}), + ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), + ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) + } const result = parseSshPtyAttachResult( - await this.mux.request('pty.attach', { - id: this.toRelayPtyId(id), - suppressReplayNotification: true, - ...(sourceRecovery ? { sourceRecovery } : {}), - ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), - ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) - }) + await this.mux.request('pty.attach', params, { timeoutMs: 10_000 }) ) this.outputState.rememberPtyIncarnation(this.toRelayPtyId(id), result.incarnationId) return result diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index ca6ab996944..a37c47f4289 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -18,6 +18,8 @@ import type { TerminalOscColorQueryReplyColors } from '../../shared/terminal-osc import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range' import type { RemoteTerminalSourceRangeConsumerHooks, + RemoteTerminalSourceRangeReplacementPublication, + RemoteTerminalSourceRangeReplacementReservation, RemoteTerminalSourceRangeStreamIdentity } from './remote-terminal-source-range-consumer' import { @@ -8887,12 +8889,37 @@ export class OrcaRuntimeService { this.remoteTerminalSourceRangeConsumerHooks?.settle(identity, ranges) } - transferRemoteTerminalSourceRanges( + reserveRemoteTerminalSourceRangeReplacement( identity: RemoteTerminalSourceRangeStreamIdentity, - ranges: readonly TerminalOutputSourceRange[], + requiredSeq: number, reason: string - ): void { - this.remoteTerminalSourceRangeConsumerHooks?.transfer(identity, ranges, reason) + ): RemoteTerminalSourceRangeReplacementReservation | null { + return ( + this.remoteTerminalSourceRangeConsumerHooks?.reserveReplacement( + identity, + requiredSeq, + reason + ) ?? null + ) + } + + commitRemoteTerminalSourceRangeReplacement( + reservation: RemoteTerminalSourceRangeReplacementReservation, + publication: RemoteTerminalSourceRangeReplacementPublication + ): boolean { + return ( + this.remoteTerminalSourceRangeConsumerHooks?.commitReplacement(reservation, publication) ?? + false + ) + } + + rollbackRemoteTerminalSourceRangeReplacement( + reservation: RemoteTerminalSourceRangeReplacementReservation, + reason: string + ): boolean { + return ( + this.remoteTerminalSourceRangeConsumerHooks?.rollbackReplacement(reservation, reason) ?? false + ) } cancelRemoteTerminalSourceRanges( diff --git a/src/main/runtime/remote-terminal-source-range-consumer.ts b/src/main/runtime/remote-terminal-source-range-consumer.ts index 280b9f3f982..3924d19bfcd 100644 --- a/src/main/runtime/remote-terminal-source-range-consumer.ts +++ b/src/main/runtime/remote-terminal-source-range-consumer.ts @@ -6,17 +6,36 @@ export type RemoteTerminalSourceRangeStreamIdentity = Readonly<{ streamGeneration: string }> +export type RemoteTerminalSourceRangeReplacementReservation = Readonly<{ + reservationId: string + identity: RemoteTerminalSourceRangeStreamIdentity + requiredSeq: number +}> + +export type RemoteTerminalSourceRangeReplacementPublication = Readonly<{ + source: 'headless' | 'renderer' + seq: number +}> + export type RemoteTerminalSourceRangeConsumerHooks = { attach: (identity: RemoteTerminalSourceRangeStreamIdentity) => boolean settle: ( identity: RemoteTerminalSourceRangeStreamIdentity, ranges: readonly TerminalOutputSourceRange[] ) => void - transfer: ( + reserveReplacement: ( identity: RemoteTerminalSourceRangeStreamIdentity, - ranges: readonly TerminalOutputSourceRange[], + requiredSeq: number, reason: string - ) => void + ) => RemoteTerminalSourceRangeReplacementReservation | null + commitReplacement: ( + reservation: RemoteTerminalSourceRangeReplacementReservation, + publication: RemoteTerminalSourceRangeReplacementPublication + ) => boolean + rollbackReplacement: ( + reservation: RemoteTerminalSourceRangeReplacementReservation, + reason: string + ) => boolean cancel: ( identity: RemoteTerminalSourceRangeStreamIdentity, ranges: readonly TerminalOutputSourceRange[], diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index b96237a4355..3c502523013 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -69,6 +69,7 @@ import { sameTerminalOutputSourceIdentity, type TerminalOutputSourceRange } from '../../../../shared/terminal-output-source-range' +import type { RemoteTerminalSourceRangeReplacementReservation } from '../../remote-terminal-source-range-consumer' const REQUESTED_SNAPSHOT_BYTE_BUDGET = 2 * 1024 * 1024 const TERMINAL_OUTPUT_FLUSH_MS = 5 @@ -124,6 +125,7 @@ type TerminalMultiplexStream = { streamGeneration: string sourceRangeLedger: TerminalSourceRangeLedger | null sourceRangeConsumerAttached: boolean + sourceRangeReplacement: RemoteTerminalSourceRangeReplacementReservation | null ackInFlightBytes: number ackWindowBytes: number supportsDesktopViewportClaims: boolean @@ -599,36 +601,45 @@ async function serializeBudgetedRequestedSnapshot( } function sendSnapshotFrames( - sendFrame: (opcode: TerminalStreamOpcode, payload?: Uint8Array) => void, + sendFrame: ( + opcode: TerminalStreamOpcode, + payload?: Uint8Array + ) => boolean | void, options: SnapshotFrameOptions -): { bytes: number; chunks: number } { - sendFrame( - TerminalStreamOpcode.SnapshotStart, - encodeTerminalStreamJson({ - kind: options.kind, - cols: options.cols, - rows: options.rows, - requestId: options.requestId, - displayMode: options.displayMode, - reason: options.reason, - seq: options.seq, - cwd: options.cwd, - source: options.source, - oscLinks: options.oscLinks, - pendingEscapeTailAnsi: options.pendingEscapeTailAnsi, - truncated: options.truncated === true, - truncatedByByteBudget: options.truncatedByByteBudget === true - }) - ) +): { bytes: number; chunks: number; published: boolean } { + if ( + sendFrame( + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + kind: options.kind, + cols: options.cols, + rows: options.rows, + requestId: options.requestId, + displayMode: options.displayMode, + reason: options.reason, + seq: options.seq, + cwd: options.cwd, + source: options.source, + oscLinks: options.oscLinks, + pendingEscapeTailAnsi: options.pendingEscapeTailAnsi, + truncated: options.truncated === true, + truncatedByByteBudget: options.truncatedByByteBudget === true + }) + ) === false + ) { + return { bytes: 0, chunks: 0, published: false } + } let chunks = 0 let bytes = 0 for (const chunk of iterateTerminalStreamTextPayloads(options.data)) { + if (sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk) === false) { + return { bytes, chunks, published: false } + } chunks++ bytes += chunk.byteLength - sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk) } - sendFrame(TerminalStreamOpcode.SnapshotEnd) - return { bytes, chunks } + const published = sendFrame(TerminalStreamOpcode.SnapshotEnd) !== false + return { bytes, chunks, published } } async function serializeBudgetedMobileSnapshot( @@ -1863,16 +1874,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ const transfer = ledger.beginTransfer() const ranges = transfer.frames.flatMap((frame) => frame.sourceRanges) try { - runtime.transferRemoteTerminalSourceRanges(identity, ranges, reason) + runtime.cancelRemoteTerminalSourceRanges(identity, ranges, reason) + } finally { transfer.commit() - } catch { - transfer.rollback() - const cancellation = ledger.beginTransfer() - try { - runtime.cancelRemoteTerminalSourceRanges(identity, ranges, `${reason}-transfer-failed`) - } finally { - cancellation.commit() - } } } const detachStream = ( @@ -2319,6 +2323,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ streamGeneration, sourceRangeLedger: sourceRangeConsumerAttached ? sourceRangeLedger : null, sourceRangeConsumerAttached, + sourceRangeReplacement: null, ackInFlightBytes: 0, ackWindowBytes: TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES, supportsDesktopViewportClaims: request.capabilities?.desktopViewportClaims === 1, @@ -2410,6 +2415,18 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ return } + const replacementIdentity = { + ptyId, + consumerId: stream.remoteDesktopSubscriptionKey, + streamGeneration: stream.streamGeneration + } + stream.sourceRangeReplacement = stream.ackOutputSourceRanges + ? runtime.reserveRemoteTerminalSourceRangeReplacement( + replacementIdentity, + runtime.getPtyOutputSequence(ptyId), + 'initial-snapshot' + ) + : null let read = await runtime.readTerminal(request.terminal) let serialized = await serializeBudgetedMobileSnapshot(runtime, ptyId, isMobile) if (closed || streams.get(request.streamId) !== stream) { @@ -2455,22 +2472,44 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ initialOutputOverflowed || (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)) }) - sendSnapshotFrames((opcode, payload) => sendFrame(request.streamId, opcode, payload), { - kind: 'scrollback', - cols: serialized?.cols ?? size?.cols ?? 80, - rows: serialized?.rows ?? size?.rows ?? 24, - displayMode, - seq: snapshotFrameSeq, - cwd: serialized?.cwd, - truncated: - initialOutputOverflowed || - (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), - truncatedByByteBudget: serialized?.truncatedByByteBudget, - source: serialized?.source, - oscLinks: serialized?.oscLinks, - pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi, - data: serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '') - }) + const snapshotPublication = sendSnapshotFrames( + (opcode, payload) => sendFrame(request.streamId, opcode, payload), + { + kind: 'scrollback', + cols: serialized?.cols ?? size?.cols ?? 80, + rows: serialized?.rows ?? size?.rows ?? 24, + displayMode, + seq: snapshotFrameSeq, + cwd: serialized?.cwd, + truncated: + initialOutputOverflowed || + (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), + truncatedByByteBudget: serialized?.truncatedByByteBudget, + source: serialized?.source, + oscLinks: serialized?.oscLinks, + pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi, + data: + serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '') + } + ) + const replacement = stream.sourceRangeReplacement + stream.sourceRangeReplacement = null + if (replacement) { + const committed = + snapshotPublication.published && + serialized?.source !== undefined && + typeof serialized.seq === 'number' && + runtime.commitRemoteTerminalSourceRangeReplacement(replacement, { + source: serialized.source, + seq: serialized.seq + }) + if (!committed) { + runtime.rollbackRemoteTerminalSourceRangeReplacement( + replacement, + 'initial-snapshot-unpublished' + ) + } + } // Why: baseline for resize re-stream gating; the client already rewrapped to these cols via the initial snapshot replay. stream.lastResizeCols = serialized?.cols ?? size?.cols stream.buffering = false diff --git a/src/main/runtime/rpc/terminal-multiplex.test.ts b/src/main/runtime/rpc/terminal-multiplex.test.ts index ceb1cf3d5a8..372fe2711a8 100644 --- a/src/main/runtime/rpc/terminal-multiplex.test.ts +++ b/src/main/runtime/rpc/terminal-multiplex.test.ts @@ -33,6 +33,7 @@ function stubRuntime(overrides: Partial = {}): OrcaRuntimeSe isPtyResizeDrivenRemotely: vi.fn().mockReturnValue(false), getRemoteDesktopFitHold: vi.fn().mockReturnValue({ mode: 'desktop-fit', cols: 120, rows: 40 }), isRemoteDesktopViewerOwner: vi.fn().mockReturnValue(false), + getPtyOutputSequence: vi.fn().mockReturnValue(0), ...overrides } as OrcaRuntimeService } @@ -138,15 +139,145 @@ function sendDesktopMultiplexSubscribe( } describe('terminal multiplex RPC', () => { + it.each(['headless', 'renderer'] as const)( + 'commits a source-range replacement only after the %s snapshot publishes', + async (source) => { + const reservation = { + reservationId: 'replacement-1', + identity: { + ptyId: 'pty-1', + consumerId: 'multiplex:conn-desktop-first-paint:7', + streamGeneration: 'generation' + }, + requiredSeq: 0 + } + const reserve = vi.fn(() => reservation) + const commit = vi.fn(() => true) + const rollback = vi.fn(() => true) + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + reserveRemoteTerminalSourceRangeReplacement: reserve, + commitRemoteTerminalSourceRangeReplacement: commit, + rollbackRemoteTerminalSourceRangeReplacement: rollback, + cancelRemoteTerminalSourceRanges: vi.fn(), + serializeTerminalBuffer: vi + .fn() + .mockResolvedValue({ data: 'snapshot', cols: 120, rows: 40, source, seq: 4 }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + + await vi.waitFor(() => expect(commit).toHaveBeenCalledOnce()) + expect(reserve).toHaveBeenCalledWith(expect.any(Object), 0, 'initial-snapshot') + expect(commit).toHaveBeenCalledWith(reservation, { source, seq: 4 }) + expect(rollback).not.toHaveBeenCalled() + const snapshotEndIndex = harness.binaryFrames.findIndex( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.SnapshotEnd + ) + expect(snapshotEndIndex).toBeGreaterThanOrEqual(0) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + } + ) + + it('rolls back replacement admission when snapshot publication is refused', async () => { + const reservation = { + reservationId: 'replacement-1', + identity: { + ptyId: 'pty-1', + consumerId: 'multiplex:conn-desktop-first-paint:7', + streamGeneration: 'generation' + }, + requiredSeq: 0 + } + const commit = vi.fn(() => true) + const rollback = vi.fn(() => true) + const harness = startDesktopMultiplexSubscribe( + { + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + reserveRemoteTerminalSourceRangeReplacement: vi.fn(() => reservation), + commitRemoteTerminalSourceRangeReplacement: commit, + rollbackRemoteTerminalSourceRangeReplacement: rollback, + cancelRemoteTerminalSourceRanges: vi.fn(), + serializeTerminalBuffer: vi.fn().mockResolvedValue({ + data: 'snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: 4 + }) + }, + undefined, + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode !== TerminalStreamOpcode.SnapshotEnd + ) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + + await harness.dispatchPromise + expect(commit).not.toHaveBeenCalled() + expect(rollback).toHaveBeenCalledWith(reservation, 'initial-snapshot-unpublished') + }) + + it('keeps legacy multiplex clients outside source replacement admission', async () => { + const attach = vi.fn(() => true) + const reserve = vi.fn() + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: attach, + reserveRemoteTerminalSourceRangeReplacement: reserve + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + + expect(attach).not.toHaveBeenCalled() + expect(reserve).not.toHaveBeenCalled() + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it('settles negotiated source ranges only at accepted encoded boundaries', async () => { let dataListener: ((data: string, meta?: RuntimeTerminalDataMeta) => void) | null = null const settle = vi.fn() - const transfer = vi.fn() + const cancel = vi.fn() const harness = startDesktopMultiplexSubscribe({ attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), settleRemoteTerminalSourceRanges: settle, - transferRemoteTerminalSourceRanges: transfer, - cancelRemoteTerminalSourceRanges: vi.fn(), + reserveRemoteTerminalSourceRangeReplacement: vi.fn(() => null), + commitRemoteTerminalSourceRangeReplacement: vi.fn(() => false), + rollbackRemoteTerminalSourceRangeReplacement: vi.fn(() => false), + cancelRemoteTerminalSourceRanges: cancel, subscribeToTerminalData: vi.fn((_ptyId, listener) => { dataListener = listener return vi.fn() @@ -263,7 +394,7 @@ describe('terminal multiplex RPC', () => { }) )! ) - expect(transfer).toHaveBeenCalledWith(expect.any(Object), [], 'stream-detached') + expect(cancel).toHaveBeenCalledWith(expect.any(Object), [], 'stream-detached') }) it.each(['refuses', 'throws'] as const)( diff --git a/src/main/ssh/relay-frame-decoder.ts b/src/main/ssh/relay-frame-decoder.ts index 3ec15d6e53e..f662bb78c63 100644 --- a/src/main/ssh/relay-frame-decoder.ts +++ b/src/main/ssh/relay-frame-decoder.ts @@ -6,7 +6,7 @@ export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 export const FRAME_DECODER_MAX_FRAMES_PER_TURN = 64 export const FRAME_DECODER_MAX_BYTES_PER_TURN = MAX_MESSAGE_SIZE + HEADER_LENGTH export const FRAME_DECODER_MAX_TURN_MS = 4, - FRAME_DECODER_MAX_RETAINED_BYTES = 2 * (MAX_MESSAGE_SIZE + HEADER_LENGTH) + FRAME_DECODER_MAX_RETAINED_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024 export class FrameDecoder { private chunks: Buffer[] = [] diff --git a/src/main/ssh/relay-protocol-backpressure.test.ts b/src/main/ssh/relay-protocol-backpressure.test.ts index 3cd42375821..2d3e742d05d 100644 --- a/src/main/ssh/relay-protocol-backpressure.test.ts +++ b/src/main/ssh/relay-protocol-backpressure.test.ts @@ -41,6 +41,27 @@ function frame(id: number, payload = `${id}`): Buffer { } describe('FrameDecoder bounded turns', () => { + it('retains at most one maximum frame plus one MiB of partial input', () => { + expect(FRAME_DECODER_MAX_RETAINED_BYTES).toBe(MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024) + const maximumFrame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(MAX_MESSAGE_SIZE)) + const acceptedError = vi.fn() + const accepted = new FrameDecoder(vi.fn(), acceptedError) + + accepted.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024)])) + + expect(acceptedError).not.toHaveBeenCalled() + expect(accepted.drain()).toHaveLength(1024 * 1024) + + const excessError = vi.fn() + const excess = new FrameDecoder(vi.fn(), excessError) + excess.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024 + 1)])) + + expect(excessError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('retained-input') }) + ) + expect(excess.drain()).toHaveLength(0) + }) + it('fails closed when one delivered chunk exceeds retained-input capacity', () => { const onError = vi.fn() const decoder = new FrameDecoder(vi.fn(), onError) diff --git a/src/main/ssh/ssh-channel-multiplexer.test.ts b/src/main/ssh/ssh-channel-multiplexer.test.ts index d46f269fd34..9002f3ca03a 100644 --- a/src/main/ssh/ssh-channel-multiplexer.test.ts +++ b/src/main/ssh/ssh-channel-multiplexer.test.ts @@ -71,6 +71,7 @@ type MuxInternals = { disposeHandlers: unknown[] lastReceivedAt: number unackedTimestamps: Map + writerSaturated: boolean } function getMuxInternals(instance: SshChannelMultiplexer): MuxInternals { @@ -330,6 +331,43 @@ describe('SshChannelMultiplexer', () => { vi.advanceTimersByTime(25_000) expect(mux.isDisposed()).toBe(true) }) + + it('suppresses false death while locally saturated and rebases both clocks on drain', () => { + mux.dispose() + let drain = (): void => {} + const written: Buffer[] = [] + const saturatedTransport: MultiplexerTransport = { + write: (data) => { + written.push(data) + return false + }, + supportsWriteSettlement: true, + onDrain: (callback) => { + drain = callback + }, + onData: vi.fn(), + onClose: vi.fn() + } + mux = new SshChannelMultiplexer(saturatedTransport) + + vi.advanceTimersByTime(5_000) + expect(getMuxInternals(mux).writerSaturated).toBe(true) + vi.advanceTimersByTime(25_000) + expect(mux.isDisposed()).toBe(false) + expect(written).toHaveLength(1) + + drain() + const resumedAt = Date.now() + const internals = getMuxInternals(mux) + expect(internals.writerSaturated).toBe(false) + expect(internals.lastReceivedAt).toBe(resumedAt) + expect(new Set(internals.unackedTimestamps.values())).toEqual(new Set([resumedAt])) + + vi.advanceTimersByTime(20_000) + expect(mux.isDisposed()).toBe(false) + vi.advanceTimersByTime(5_000) + expect(mux.isDisposed()).toBe(true) + }) }) describe('wake guard (timer pause across system sleep, #7773)', () => { diff --git a/src/main/ssh/ssh-channel-multiplexer.ts b/src/main/ssh/ssh-channel-multiplexer.ts index 686659c3de5..3e7d55d19db 100644 --- a/src/main/ssh/ssh-channel-multiplexer.ts +++ b/src/main/ssh/ssh-channel-multiplexer.ts @@ -61,6 +61,7 @@ export class SshChannelMultiplexer { private connectionHealthTimer: ReturnType | null = null private disposed = false private decoderReadPaused = false + private writerSaturated = false // Track the oldest unacked outgoing message timestamp private unackedTimestamps = new Map() @@ -71,8 +72,10 @@ export class SshChannelMultiplexer { constructor(transport: MultiplexerTransport) { this.transport = transport - this.writer = new SshMultiplexerTransportWriter(transport, (error) => - this.handleProtocolError(error) + this.writer = new SshMultiplexerTransportWriter( + transport, + (error) => this.handleProtocolError(error), + (saturated) => this.handleWriterSaturationChange(saturated) ) this.decoder = new FrameDecoder( @@ -369,10 +372,13 @@ export class SshChannelMultiplexer { if (this.disposed) { return } - const seq = this.nextOutgoingSeq++ + const seq = this.nextOutgoingSeq const frame = encodeKeepAliveFrame(seq, this.highestReceivedSeq) + if (!this.writer.enqueue(frame, 'liveness')) { + return + } + this.nextOutgoingSeq++ this.trackOutgoingTimestamp(seq, true) - this.writer.enqueue(frame, 'control') } private handleFrame(frame: DecodedFrame): void { @@ -528,7 +534,7 @@ export class SshChannelMultiplexer { this.sendKeepAlive() - if (this.disposed || resumedAfterWake || this.decoderReadPaused) { + if (this.disposed || resumedAfterWake || this.decoderReadPaused || this.writerSaturated) { return } @@ -590,6 +596,13 @@ export class SshChannelMultiplexer { } } + private handleWriterSaturationChange(saturated: boolean): void { + this.writerSaturated = saturated + if (!saturated && !this.disposed) { + this.rebaseHealthClocks(Date.now()) + } + } + private rebaseHealthClocks(now: number): void { this.lastReceivedAt = now for (const seq of this.unackedTimestamps.keys()) { diff --git a/src/main/ssh/ssh-multiplexer-transport-writer.test.ts b/src/main/ssh/ssh-multiplexer-transport-writer.test.ts index 7087fbff5ad..649af6d04d8 100644 --- a/src/main/ssh/ssh-multiplexer-transport-writer.test.ts +++ b/src/main/ssh/ssh-multiplexer-transport-writer.test.ts @@ -70,6 +70,44 @@ describe('SshMultiplexerTransportWriter', () => { expect(failed).not.toHaveBeenCalled() }) + it('allows one coalesced liveness bypass per saturated write settlement', () => { + const harness = transportHarness([false, true, true, true]) + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn()) + + writer.enqueue(Buffer.from('ordinary-1'), 'ordinary') + writer.enqueue(Buffer.from('ordinary-2'), 'ordinary') + expect(writer.enqueue(Buffer.from('liveness-1'), 'liveness')).toBe(true) + expect(writer.enqueue(Buffer.from('liveness-coalesced'), 'liveness')).toBe(false) + expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1']) + + harness.callbacks[1]({ ok: true }) + expect(writer.enqueue(Buffer.from('liveness-2'), 'liveness')).toBe(true) + expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1', 'liveness-2']) + + harness.drain() + expect(harness.writes.map(String)).toEqual([ + 'ordinary-1', + 'liveness-1', + 'liveness-2', + 'ordinary-2' + ]) + writer.dispose() + }) + + it('reports saturated epochs and their drain boundary exactly once', () => { + const harness = transportHarness([false]) + const saturation = vi.fn() + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn(), saturation) + + writer.enqueue(Buffer.from('ordinary'), 'ordinary') + writer.enqueue(Buffer.from('queued'), 'ordinary') + expect(saturation.mock.calls).toEqual([[true]]) + + harness.drain() + expect(saturation.mock.calls).toEqual([[true], [false]]) + writer.dispose() + }) + it('keeps a full ordinary lane from consuming the control reserve', () => { const harness = transportHarness([false, true, true]) const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn()) diff --git a/src/main/ssh/ssh-multiplexer-transport-writer.ts b/src/main/ssh/ssh-multiplexer-transport-writer.ts index bb9d45b1068..287ad4e0fcb 100644 --- a/src/main/ssh/ssh-multiplexer-transport-writer.ts +++ b/src/main/ssh/ssh-multiplexer-transport-writer.ts @@ -13,7 +13,7 @@ export type MultiplexerTransport = { close?: () => void } -export type MultiplexerWriterLane = 'ordinary' | 'control' +export type MultiplexerWriterLane = 'ordinary' | 'control' | 'liveness' type WriterEntry = { data: Buffer @@ -53,11 +53,13 @@ export class SshMultiplexerTransportWriter { private drainObservedDuringWrite = false private pumping = false private closed = false + private livenessOutstanding = false private removeDrainListener: (() => void) | null = null constructor( private readonly transport: MultiplexerTransport, - private readonly onFailure: (error: Error) => void + private readonly onFailure: (error: Error) => void, + private readonly onSaturationChange: (saturated: boolean) => void = () => {} ) { if (transport.onDrain) { const remove = transport.onDrain(() => this.handleDrain()) @@ -75,6 +77,9 @@ export class SshMultiplexerTransportWriter { settle({ ok: false, error: new Error('Multiplexer writer is closed') }) return false } + if (lane === 'liveness' && this.livenessOutstanding) { + return false + } const admissionError = this.admissionError(data.length, lane) if (admissionError) { settle({ ok: false, error: admissionError }) @@ -83,8 +88,12 @@ export class SshMultiplexerTransportWriter { } const entry = { data, lane, onSettled: settle, settled: false } this.retain(entry) - this.queue.push(entry) - this.pump() + if (lane === 'liveness' && this.saturated) { + this.writeEntry(entry) + } else { + this.queue.push(entry) + this.pump() + } return true } @@ -156,7 +165,7 @@ export class SshMultiplexerTransportWriter { if (!this.transport.onDrain) { throw new Error('Multiplexer transport returned write(false) without drain support') } - this.saturated = !this.drainObservedDuringWrite + this.setSaturated(!this.drainObservedDuringWrite) if (this.transport.supportsWriteSettlement !== true && this.saturated) { this.settleOnDrain.add(entry) } else if (this.transport.supportsWriteSettlement !== true) { @@ -201,7 +210,7 @@ export class SshMultiplexerTransportWriter { if (!this.saturated) { return } - this.saturated = false + this.setSaturated(false) for (const entry of Array.from(this.settleOnDrain)) { this.release(entry, { ok: true }) } @@ -217,6 +226,9 @@ export class SshMultiplexerTransportWriter { this.controlBytes += entry.data.length this.controlFrames++ } + if (entry.lane === 'liveness') { + this.livenessOutstanding = true + } } private release(entry: WriterEntry, result: MultiplexerWriteSettlement): void { @@ -233,9 +245,20 @@ export class SshMultiplexerTransportWriter { this.controlBytes -= entry.data.length this.controlFrames-- } + if (entry.lane === 'liveness') { + this.livenessOutstanding = false + } entry.onSettled(result) } + private setSaturated(saturated: boolean): void { + if (this.saturated === saturated) { + return + } + this.saturated = saturated + this.onSaturationChange(saturated) + } + private fail(error: Error): void { if (this.closed) { return diff --git a/src/main/ssh/ssh-pty-recovery-retention-budget.test.ts b/src/main/ssh/ssh-pty-recovery-retention-budget.test.ts new file mode 100644 index 00000000000..47219e61780 --- /dev/null +++ b/src/main/ssh/ssh-pty-recovery-retention-budget.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget' + +describe('SshPtyRecoveryRetentionBudget', () => { + it('bounds fragmented recovery by per-PTY and session source, charged bytes, and frames', () => { + const budget = new SshPtyRecoveryRetentionBudget({ + perPtySourceSu: 4, + perPtyBytes: 300, + perPtyFrames: 2, + sessionSourceSu: 6, + sessionBytes: 500, + sessionFrames: 3 + }) + + expect(budget.tryRetain('pty-1', 'aa', 2)).toBe(true) + expect(budget.tryRetain('pty-1', 'bb', 2)).toBe(true) + expect(budget.tryRetain('pty-1', 'c', 0)).toBe(false) + expect(budget.tryRetain('pty-2', 'cc', 2)).toBe(true) + expect(budget.snapshot()).toEqual({ sourceSu: 6, bytes: 396, frames: 3, ptys: 2 }) + expect(budget.tryRetain('pty-2', 'd', 1)).toBe(false) + + budget.release('pty-1') + + expect(budget.snapshot()).toEqual({ sourceSu: 2, bytes: 132, frames: 1, ptys: 1 }) + expect(budget.tryRetain('pty-2', 'dd', 2)).toBe(true) + }) + + it('charges UTF-16 storage plus one record before aggregate admission', () => { + const budget = new SshPtyRecoveryRetentionBudget({ + perPtySourceSu: 100, + perPtyBytes: 1_000, + perPtyFrames: 10, + sessionSourceSu: 100, + sessionBytes: 500, + sessionFrames: 10 + }) + + expect(budget.tryRetain('pty-1', '\u0000'.repeat(64), 1)).toBe(true) + expect(budget.snapshot().bytes).toBe(256) + expect(budget.tryRetain('pty-2', '\u0000'.repeat(64), 1)).toBe(false) + expect(budget.snapshot()).toEqual({ sourceSu: 1, bytes: 256, frames: 1, ptys: 1 }) + }) +}) diff --git a/src/main/ssh/ssh-pty-recovery-retention-budget.ts b/src/main/ssh/ssh-pty-recovery-retention-budget.ts new file mode 100644 index 00000000000..873b56a8778 --- /dev/null +++ b/src/main/ssh/ssh-pty-recovery-retention-budget.ts @@ -0,0 +1,95 @@ +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import { chargedPtyRetainedStringBytes } from '../../shared/pty-retained-string-memory' + +export const SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU = DEFAULT_PTY_SOURCE_WINDOW_SU +export const SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES = 2 * 1024 * 1024 +export const SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES = 1_024 +export const SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU = 50 * DEFAULT_PTY_SOURCE_WINDOW_SU +export const SSH_PTY_RECOVERY_SESSION_MAX_BYTES = 64 * 1024 * 1024 +export const SSH_PTY_RECOVERY_SESSION_MAX_FRAMES = 64 * 1_024 + +export type SshPtyRecoveryRetentionLimits = Readonly<{ + perPtySourceSu: number + perPtyBytes: number + perPtyFrames: number + sessionSourceSu: number + sessionBytes: number + sessionFrames: number +}> + +type RetainedPty = { + sourceSu: number + bytes: number + frames: number +} + +const DEFAULT_LIMITS: SshPtyRecoveryRetentionLimits = Object.freeze({ + perPtySourceSu: SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU, + perPtyBytes: SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES, + perPtyFrames: SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES, + sessionSourceSu: SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU, + sessionBytes: SSH_PTY_RECOVERY_SESSION_MAX_BYTES, + sessionFrames: SSH_PTY_RECOVERY_SESSION_MAX_FRAMES +}) + +export class SshPtyRecoveryRetentionBudget { + private readonly retainedByPty = new Map() + private sourceSu = 0 + private bytes = 0 + private frames = 0 + + constructor(private readonly limits: SshPtyRecoveryRetentionLimits = DEFAULT_LIMITS) {} + + tryRetain(ptyId: string, data: string, sourceSu: number): boolean { + if (!Number.isSafeInteger(sourceSu) || sourceSu < 0) { + return false + } + const retained = this.retainedByPty.get(ptyId) ?? { sourceSu: 0, bytes: 0, frames: 0 } + const chargedBytes = chargedPtyRetainedStringBytes(data) + if ( + retained.sourceSu + sourceSu > this.limits.perPtySourceSu || + retained.bytes + chargedBytes > this.limits.perPtyBytes || + retained.frames + 1 > this.limits.perPtyFrames || + this.sourceSu + sourceSu > this.limits.sessionSourceSu || + this.bytes + chargedBytes > this.limits.sessionBytes || + this.frames + 1 > this.limits.sessionFrames + ) { + return false + } + retained.sourceSu += sourceSu + retained.bytes += chargedBytes + retained.frames++ + this.retainedByPty.set(ptyId, retained) + this.sourceSu += sourceSu + this.bytes += chargedBytes + this.frames++ + return true + } + + release(ptyId: string): void { + const retained = this.retainedByPty.get(ptyId) + if (!retained) { + return + } + this.retainedByPty.delete(ptyId) + this.sourceSu -= retained.sourceSu + this.bytes -= retained.bytes + this.frames -= retained.frames + } + + clear(): void { + this.retainedByPty.clear() + this.sourceSu = 0 + this.bytes = 0 + this.frames = 0 + } + + snapshot(): Readonly<{ sourceSu: number; bytes: number; frames: number; ptys: number }> { + return Object.freeze({ + sourceSu: this.sourceSu, + bytes: this.bytes, + frames: this.frames, + ptys: this.retainedByPty.size + }) + } +} diff --git a/src/main/ssh/ssh-pty-source-credit-rollout.test.ts b/src/main/ssh/ssh-pty-source-credit-rollout.test.ts index 7a92eb0684e..06b7f863989 100644 --- a/src/main/ssh/ssh-pty-source-credit-rollout.test.ts +++ b/src/main/ssh/ssh-pty-source-credit-rollout.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest' import { + assertSshPtySourceCreditRelayLaunchPolicy, isSshPtySourceCreditV1Enabled, + SshPtySourceCreditRestartRequiredError, SSH_PTY_SOURCE_CREDIT_V1_ENV } from './ssh-pty-source-credit-rollout' @@ -11,4 +13,39 @@ describe('SSH PTY source-credit rollout', () => { expect(isSshPtySourceCreditV1Enabled({ [SSH_PTY_SOURCE_CREDIT_V1_ENV]: 'true' })).toBe(false) expect(isSshPtySourceCreditV1Enabled({ [SSH_PTY_SOURCE_CREDIT_V1_ENV]: '1' })).toBe(true) }) + + it('surfaces a deliberate restart action for a live off-to-on transition', () => { + expect(new SshPtySourceCreditRestartRequiredError()).toMatchObject({ + name: 'SshPtySourceCreditRestartRequiredError', + code: 'ssh_pty_source_credit_restart_required', + message: expect.stringContaining('Reset the relay') + }) + }) + + it('permits stable gate-off legacy and stable gate-on V1 startup selections', () => { + expect(() => assertSshPtySourceCreditRelayLaunchPolicy(false, 'off')).not.toThrow() + expect(() => assertSshPtySourceCreditRelayLaunchPolicy(false, 'v1')).not.toThrow() + expect(() => assertSshPtySourceCreditRelayLaunchPolicy(true, 'v1')).not.toThrow() + }) + + it.each(['off', '', 'unknown'])('fails closed when gate-on reaches policy %j', (policy) => { + expect(() => assertSshPtySourceCreditRelayLaunchPolicy(true, policy)).toThrow( + SshPtySourceCreditRestartRequiredError + ) + }) + + it('keeps the process-start selection stable across later environment mutation', () => { + const startupSelection = isSshPtySourceCreditV1Enabled() + const previous = process.env[SSH_PTY_SOURCE_CREDIT_V1_ENV] + process.env[SSH_PTY_SOURCE_CREDIT_V1_ENV] = startupSelection ? '0' : '1' + try { + expect(isSshPtySourceCreditV1Enabled()).toBe(startupSelection) + } finally { + if (previous === undefined) { + delete process.env[SSH_PTY_SOURCE_CREDIT_V1_ENV] + } else { + process.env[SSH_PTY_SOURCE_CREDIT_V1_ENV] = previous + } + } + }) }) diff --git a/src/main/ssh/ssh-pty-source-credit-rollout.ts b/src/main/ssh/ssh-pty-source-credit-rollout.ts index 9355ac3be52..34bb316266e 100644 --- a/src/main/ssh/ssh-pty-source-credit-rollout.ts +++ b/src/main/ssh/ssh-pty-source-credit-rollout.ts @@ -1,5 +1,31 @@ export const SSH_PTY_SOURCE_CREDIT_V1_ENV = 'ORCA_SSH_PTY_SOURCE_CREDIT_V1' -export function isSshPtySourceCreditV1Enabled(env: NodeJS.ProcessEnv = process.env): boolean { +function readSshPtySourceCreditV1Selection(env: NodeJS.ProcessEnv): boolean { return env[SSH_PTY_SOURCE_CREDIT_V1_ENV] === '1' } + +const SSH_PTY_SOURCE_CREDIT_V1_STARTUP_SELECTION = readSshPtySourceCreditV1Selection(process.env) + +export function isSshPtySourceCreditV1Enabled(env?: NodeJS.ProcessEnv): boolean { + return env ? readSshPtySourceCreditV1Selection(env) : SSH_PTY_SOURCE_CREDIT_V1_STARTUP_SELECTION +} + +export class SshPtySourceCreditRestartRequiredError extends Error { + readonly code = 'ssh_pty_source_credit_restart_required' + + constructor() { + super( + 'SSH PTY source credit was enabled after this detached relay started. Reset the relay, then reconnect to launch it with source-credit support.' + ) + this.name = 'SshPtySourceCreditRestartRequiredError' + } +} + +export function assertSshPtySourceCreditRelayLaunchPolicy( + enabled: boolean, + relayPolicy: string +): void { + if (enabled && relayPolicy !== 'v1') { + throw new SshPtySourceCreditRestartRequiredError() + } +} diff --git a/src/main/ssh/ssh-relay-deploy.test.ts b/src/main/ssh/ssh-relay-deploy.test.ts index bbf8d0a6e0f..4856c271439 100644 --- a/src/main/ssh/ssh-relay-deploy.test.ts +++ b/src/main/ssh/ssh-relay-deploy.test.ts @@ -484,6 +484,7 @@ describe('deployAndLaunchRelay', () => { expect(launchCommand).toContain(`--grace-time ${DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS}`) expect(launchCommand).not.toContain('--pty-source-credit-v1') + expect(launchCommand).toContain("printf '%s\\n' 'off'") }) it('allows an unlimited SSH disconnect grace window', async () => { @@ -504,6 +505,7 @@ describe('deployAndLaunchRelay', () => { expect(launchCommand).toContain('--grace-time 0') expect(launchCommand).toContain('--pty-source-credit-v1') + expect(launchCommand).toContain("printf '%s\\n' 'v1'") }) it('clamps configured SSH disconnect grace to the seven-day maximum', async () => { @@ -770,6 +772,8 @@ describe('deployAndLaunchRelay', () => { ) expect(launchScript).toContain('--endpoint-dir') expect(launchScript).toContain('--pty-source-credit-v1') + expect(launchScript).toContain('.pty-source-credit-policy') + expect(launchScript).toContain("-Value 'v1'") expect(launchScript).not.toContain('\\\\.\\pipe\\agent-hooks') const waitScript = decodedScripts.find((script) => script.includes('deadline=Date.now()')) ?? '' expect(waitScript).toContain('setTimeout(attempt,intervalMs)') diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index fd28f5badb0..6cbb577a968 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -54,6 +54,7 @@ import { isWindowsRemoteHost, joinRemotePath, normalizeRemoteHome, + remoteBasename, validateRemoteHome, type RemoteHostPlatform } from './ssh-remote-platform' @@ -73,6 +74,10 @@ import { MAX_SSH_RELAY_GRACE_PERIOD_SECONDS, MIN_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../../shared/ssh-types' +import { + assertSshPtySourceCreditRelayLaunchPolicy, + SshPtySourceCreditRestartRequiredError +} from './ssh-pty-source-credit-rollout' export type RelayDeployResult = { transport: MultiplexerTransport @@ -108,6 +113,37 @@ function execHostCommand( }) } +function sourceCreditPolicyPath( + hostPlatform: RemoteHostPlatform, + remoteDir: string, + endpoint: string +): string { + return joinRemotePath( + hostPlatform, + remoteDir, + `${remoteBasename(endpoint, hostPlatform)}.pty-source-credit-policy` + ) +} + +async function assertExistingRelaySourceCreditPolicy( + conn: SshConnection, + hostPlatform: RemoteHostPlatform, + policyPath: string, + enabled: boolean, + signal?: AbortSignal +): Promise { + if (!enabled) { + return + } + const command = isWindowsRemoteHost(hostPlatform) + ? powerShellCommand( + `if (Test-Path -LiteralPath ${powerShellLiteral(policyPath)} -PathType Leaf) { Get-Content -LiteralPath ${powerShellLiteral(policyPath)} -Raw -ErrorAction SilentlyContinue }` + ) + : `test -f ${shellEscape(policyPath)} && cat ${shellEscape(policyPath)} || true` + const policy = (await execHostCommand(conn, hostPlatform, command, { signal })).trim() + assertSshPtySourceCreditRelayLaunchPolicy(enabled, policy) +} + /** * Deploy the relay to the remote host and launch it, returning the transport (relay's stdin/stdout) for multiplexer use. */ @@ -1191,18 +1227,21 @@ async function launchRelay( const sockFile = relayEndpointForHost(hostPlatform, remoteDir, sockName) const endpointDir = relayHookEndpointDirForHost(hostPlatform, remoteDir, sockFile) const credentialFile = joinRemotePath(hostPlatform, remoteDir, `${sockName}.credential`) + const sourceCreditPolicyFile = sourceCreditPolicyPath(hostPlatform, remoteDir, sockFile) if (isWindowsRemoteHost(hostPlatform)) { const activePipeMarkerPath = windowsActivePipeMarkerPath(hostPlatform, remoteDir, sockName) - const activeEndpoint = (await readWindowsActiveRelayEndpoint( + const discoveredActiveEndpoint = await readWindowsActiveRelayEndpoint( conn, hostPlatform, remoteDir, activePipeMarkerPath, signal - )) ?? { + ) + const activeEndpoint = discoveredActiveEndpoint ?? { sockPath: sockFile, - endpointDir + endpointDir, + sourceCreditPolicyFile } const fallbackEndpoint = buildWindowsRelayFallbackEndpoint(hostPlatform, remoteDir, sockName) const launched = await launchWindowsRelay( @@ -1213,6 +1252,7 @@ async function launchRelay( nodePath, sockPath: activeEndpoint.sockPath, endpointDir: activeEndpoint.endpointDir, + sourceCreditPolicyFile: activeEndpoint.sourceCreditPolicyFile, graceTime, activePipeMarkerPath, reconnectFallback: fallbackEndpoint, @@ -1233,6 +1273,13 @@ async function launchRelay( ) console.warn(`[ssh-relay] Socket probe result: "${probeOutput.trim()}"`) if (probeOutput.trim() === 'ALIVE') { + await assertExistingRelaySourceCreditPolicy( + conn, + hostPlatform, + sourceCreditPolicyFile, + enablePtySourceCreditV1, + signal + ) console.log('[ssh-relay] Existing relay socket found, attempting reconnect...') try { const channel = await conn.exec( @@ -1260,6 +1307,9 @@ async function launchRelay( } } } catch (err) { + if (err instanceof SshPtySourceCreditRestartRequiredError) { + throw err + } if (isUnconfirmedSshCommandTermination(err)) { throw err } @@ -1273,7 +1323,8 @@ async function launchRelay( await writeRelayEndpointCredential(conn, hostPlatform, credentialFile, signal) // Why: --log-file lets the relay rotate relay.log in-process; the shell redirect stays to capture pre-JS boot/crash output. const sourceCreditFlag = enablePtySourceCreditV1 ? ' --pty-source-credit-v1' : '' - const launchCmd = `cd ${escapedDir} && chmod 600 ${shellEscape(credentialFile)} && nohup ${escapedNode} relay.js --detached${sourceCreditFlag} --grace-time ${graceTime} --sock-path ${shellEscape(sockFile)} --credential-file ${shellEscape(credentialFile)} --log-file ${shellEscape(logFile)} > ${shellEscape(logFile)} 2>&1 ${shellEscape(sourceCreditPolicyFile)} && chmod 600 ${shellEscape(credentialFile)} ${shellEscape(sourceCreditPolicyFile)} && nohup ${escapedNode} relay.js --detached${sourceCreditFlag} --grace-time ${graceTime} --sock-path ${shellEscape(sockFile)} --credential-file ${shellEscape(credentialFile)} --log-file ${shellEscape(logFile)} > ${shellEscape(logFile)} 2>&1 {}) launchChannel.on('error', () => {}) @@ -1371,7 +1422,8 @@ function buildWindowsRelayFallbackEndpoint( const sockPath = relayEndpointForHost(hostPlatform, remoteDir, fallbackSockName) return { sockPath, - endpointDir: relayHookEndpointDirForHost(hostPlatform, remoteDir, sockPath) + endpointDir: relayHookEndpointDirForHost(hostPlatform, remoteDir, sockPath), + sourceCreditPolicyFile: sourceCreditPolicyPath(hostPlatform, remoteDir, sockPath) } } @@ -1399,7 +1451,8 @@ async function readWindowsActiveRelayEndpoint( } return { sockPath, - endpointDir: relayHookEndpointDirForHost(hostPlatform, remoteDir, sockPath) + endpointDir: relayHookEndpointDirForHost(hostPlatform, remoteDir, sockPath), + sourceCreditPolicyFile: sourceCreditPolicyPath(hostPlatform, remoteDir, sockPath) } } @@ -1429,6 +1482,7 @@ async function rememberWindowsActiveRelayEndpoint( type WindowsRelayEndpoint = { sockPath: string endpointDir: string + sourceCreditPolicyFile: string } type WindowsRelayLaunchOptions = { @@ -1450,6 +1504,13 @@ async function launchWindowsRelay( ): Promise<{ transport: MultiplexerTransport; nodePath: string; sockPath: string }> { let launchOpts = opts if ((await probeWindowsRelayPipe(conn, hostPlatform, opts, signal)) === 'READY') { + await assertExistingRelaySourceCreditPolicy( + conn, + hostPlatform, + opts.sourceCreditPolicyFile, + opts.enablePtySourceCreditV1, + signal + ) try { const transport = await connectWindowsRelay(conn, hostPlatform, opts, signal) await rememberWindowsActiveRelayEndpoint( @@ -1482,6 +1543,13 @@ async function launchWindowsRelay( launchOpts !== opts && (await probeWindowsRelayPipe(conn, hostPlatform, launchOpts, signal)) === 'READY' ) { + await assertExistingRelaySourceCreditPolicy( + conn, + hostPlatform, + launchOpts.sourceCreditPolicyFile, + launchOpts.enablePtySourceCreditV1, + signal + ) try { const transport = await connectWindowsRelay(conn, hostPlatform, launchOpts, signal) await rememberWindowsActiveRelayEndpoint( @@ -1521,6 +1589,7 @@ async function launchWindowsRelay( logFile, errFile, launchOpts.credentialFile, + launchOpts.sourceCreditPolicyFile, launchOpts.enablePtySourceCreditV1 ), { signal } @@ -1614,6 +1683,7 @@ function windowsRelayLaunchCommand( logFile: string, errFile: string, credentialFile: string, + sourceCreditPolicyFile: string, enablePtySourceCreditV1 = false ): string { const relayScript = joinRemotePath(hostPlatform, remoteDir, 'relay.js') @@ -1644,7 +1714,9 @@ function windowsRelayLaunchCommand( nodePath, remoteDir, [ + `Set-Content -LiteralPath ${powerShellLiteral(sourceCreditPolicyFile)} -Value ${powerShellLiteral(enablePtySourceCreditV1 ? 'v1' : 'off')} -NoNewline`, `& icacls.exe ${powerShellLiteral(credentialFile)} /inheritance:r /grant:r "$($env:USERNAME):(R,W)" | Out-Null`, + `& icacls.exe ${powerShellLiteral(sourceCreditPolicyFile)} /inheritance:r /grant:r "$($env:USERNAME):(R,W)" | Out-Null`, `$result = Invoke-CimMethod -ClassName Win32_Process -MethodName Create -Arguments @{ CommandLine = ${powerShellLiteral(wmiCommandLine)}; CurrentDirectory = ${powerShellLiteral(remoteDir)} }`, `if ($result.ReturnValue -ne 0) { throw "Win32_Process.Create failed with $($result.ReturnValue)" }` ].join('; ') diff --git a/src/main/ssh/ssh-relay-session-data-delivery.test.ts b/src/main/ssh/ssh-relay-session-data-delivery.test.ts index 1f7e811fb98..103136dbc7e 100644 --- a/src/main/ssh/ssh-relay-session-data-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-data-delivery.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session' import { SshRelaySession } from './ssh-relay-session' import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' @@ -34,7 +35,9 @@ vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ allocateSshPtyProviderGeneration: vi.fn(() => 23), closeSshPtyOutputGeneration: vi.fn(), getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), - installSshPtySourceAckPublisher: vi.fn(() => () => {}) + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}), + applySshPtySourceCancellationProof: vi.fn() })) vi.mock('./ssh-channel-multiplexer', () => ({ @@ -105,9 +108,17 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ unregisterSshGitProvider: vi.fn() })) -const { getSshPtyProvider, getPtyIdsForConnection, registerSshPtyProvider } = - await import('../ipc/pty') +const { + clearProviderPtyState, + clearPtyOwnershipForConnection, + deletePtyOwnership, + getSshPtyProvider, + getPtyIdsForConnection, + registerSshPtyProvider, + setPtyOwnership +} = await import('../ipc/pty') const { closeSshPtyOutputGeneration } = await import('../ipc/ssh-pty-output-intake-registry') +const { applySshPtySourceCancellationProof } = await import('../ipc/ssh-pty-output-intake-registry') const { getSshPtyAcceptedSourceCheckpoints } = await import('../ipc/ssh-pty-output-intake-registry') const { installSshPtySourceAckPublisher } = await import('../ipc/ssh-pty-output-intake-registry') const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') @@ -117,6 +128,8 @@ describe('SshRelaySession data delivery', () => { vi.clearAllMocks() ptyDataHandlerRef.current = undefined attachForReconnectMock.mockResolvedValue({}) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([]) openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ mode: 'negotiated', clientInstanceId: options.clientInstanceId, @@ -133,6 +146,122 @@ describe('SshRelaySession data delivery', () => { mockDeploySuccess() }) + async function runRecoverySequence(args: { + targetId: string + recoveryEndSu: number + recoveryFrame?: readonly [startSu: number, endSu: number] + liveFrame?: readonly [startSu: number, endSu: number] + }) { + muxRequestMock.mockImplementation(async (method) => + method === 'pty.cancelDelivery' + ? { canceled: true, sentEndSu: args.recoveryEndSu, creditedEndSu: 4 } + : [] + ) + let generation = 0 + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: ++generation, + ownerGeneration: generation, + ownerLease: `owner-lease-${generation}`, + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: `ssh:${args.targetId}@@pty-1`, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const deps = createMockDeps() + const session = new SshRelaySession( + args.targetId, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward, + undefined, + undefined, + () => true + ) + await session.establish(deps.mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue([`ssh:${args.targetId}@@pty-1`]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + attachForReconnectMock.mockImplementation(async () => { + queueMicrotask(() => { + if (args.recoveryFrame) { + const [sourceStartSu, sourceEndSu] = args.recoveryFrame + ptyDataHandlerRef.current?.({ + id: `ssh:${args.targetId}@@pty-1`, + data: 'recovery', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: sourceEndSu - sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `new-token:${sourceStartSu}:${sourceEndSu}`, + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu, + sourceEndSu + } + }) + } + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: args.recoveryEndSu + }) + if (args.liveFrame) { + const [sourceStartSu, sourceEndSu] = args.liveFrame + ptyDataHandlerRef.current?.({ + id: `ssh:${args.targetId}@@pty-1`, + data: 'live', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: sourceEndSu - sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `new-token:${sourceStartSu}:${sourceEndSu}`, + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu, + sourceEndSu + } + }) + } + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: args.recoveryEndSu + } + } + }) + await session.reconnect(deps.mockConn) + return { ...deps, session } + } + it('transfers negotiated owner recovery exactly once across an explicit detach', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ @@ -163,6 +292,86 @@ describe('SshRelaySession data delivery', () => { fresh.dispose() }) + it('clears stale owner recovery and retries a fresh same-build relay once without resume', async () => { + const targetId = 'fresh-relay-retry' + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, + platform: 'linux-x64', + serverBuildId: 'test-relay-build' + }) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: `ssh:${targetId}@@pty-1`, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const first = new SshRelaySession( + targetId, + getMainWindow, + mockStore, + mockPortForward, + undefined, + undefined, + () => true + ) + await first.establish(mockConn) + first.detach() + + openConsumerSessionMock + .mockRejectedValueOnce( + Object.assign(new Error('Owner recovery lease is stale'), { + code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR + }) + ) + .mockImplementationOnce(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'fresh-owner-lease', + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + vi.mocked(getPtyIdsForConnection).mockReturnValue([`ssh:${targetId}@@pty-1`]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + attachForReconnectMock.mockResolvedValue({ + incarnationId: 'incarnation-1', + sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' } + }) + const second = new SshRelaySession( + targetId, + getMainWindow, + mockStore, + mockPortForward, + undefined, + undefined, + () => true + ) + const openCallCountBeforeRetry = openConsumerSessionMock.mock.calls.length + + await second.establish(mockConn) + + const retryCalls = openConsumerSessionMock.mock.calls + .slice(openCallCountBeforeRetry) + .map(([, options]) => options) + expect(retryCalls).toHaveLength(2) + expect(retryCalls[0]).toHaveProperty('resume') + expect(retryCalls[1]).not.toHaveProperty('resume') + expect(attachForReconnectMock).toHaveBeenCalledWith( + 'pty-1', + undefined, + Object.freeze({ status: 'checkpointUnavailable' }) + ) + second.dispose() + }) + it('delivers empty transformed relay spans with raw sequence metadata', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const runtime = { onPtyData: vi.fn(() => 17), onPtyExit: vi.fn() } @@ -474,17 +683,33 @@ describe('SshRelaySession data delivery', () => { queueMicrotask(() => { ptyDataHandlerRef.current?.({ id: 'ssh:target-1@@pty-1', - data: 'reco', + data: 're', providerGeneration: 23, ptyIncarnation: 'incarnation-1', - sequenceChars: 4, + sequenceChars: 2, source: { relayPtyId: 'pty-1', - spanId: 'new-token:4:8', + spanId: 'new-token:4:6', clientGeneration: 2, ownerGeneration: 2, deliveryToken: 'new-token', sourceStartSu: 4, + sourceEndSu: 6 + } + }) + ptyDataHandlerRef.current?.({ + id: 'ssh:target-1@@pty-1', + data: 'co', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: 2, + source: { + relayPtyId: 'pty-1', + spanId: 'new-token:6:8', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu: 6, sourceEndSu: 8 } }) @@ -543,7 +768,8 @@ describe('SshRelaySession data delivery', () => { }) ) expect(acceptOutputDataMock.mock.calls.map(([payload]) => payload.data)).toEqual([ - 'reco', + 're', + 'co', 'live' ]) expect(transferDisposedMux).toBe(false) @@ -562,4 +788,59 @@ describe('SshRelaySession data delivery', () => { }) expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount) }) + + it.each([ + ['gap', [5, 8] as const], + ['overlap', [3, 8] as const], + ['incomplete suffix', [4, 6] as const], + ['missing body', undefined] + ])('rejects %s recovery without destroying the physical PTY or lease', async (label, frame) => { + const targetId = `invalid-recovery-${label.replace(' ', '-')}` + const { mockStore, mockWindow } = await runRecoverySequence({ + targetId, + recoveryEndSu: 8, + ...(frame ? { recoveryFrame: frame } : {}) + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(muxRequestMock).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token' + }) + expect(applySshPtySourceCancellationProof).toHaveBeenCalledWith( + { + id: `ssh:${targetId}@@pty-1`, + code: -1, + providerGeneration: 23, + ptyIncarnation: 'incarnation-1' + }, + { sentEndSu: 8, creditedEndSu: 4 } + ) + expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'detached') + expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith(targetId, 'pty-1', 'expired') + expect(mockStore.markSshRemotePtyLeases).not.toHaveBeenCalled() + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(setPtyOwnership).not.toHaveBeenCalled() + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + }) + + it('accepts empty recovery only when the checkpoint equals the recovery end', async () => { + const { mockStore } = await runRecoverySequence({ + targetId: 'empty-recovery', + recoveryEndSu: 4, + liveFrame: [4, 8] + }) + + expect(acceptOutputDataMock.mock.calls.map(([payload]) => payload.data)).toEqual(['live']) + expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( + 'empty-recovery', + 'pty-1', + 'attached' + ) + }) }) diff --git a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts index 1a7159da4e3..db7abff3939 100644 --- a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts @@ -138,6 +138,72 @@ describe('SshRelaySession reconnect incarnation ordering', () => { vi.mocked(getPtyIdsForConnection).mockReturnValue([]) }) + it('bounds fifty reattaches to eight workers without slow or failed sibling head-of-line delay', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + vi.clearAllMocks() + mockDeploySuccess() + + const ptyIds = Array.from({ length: 50 }, (_, index) => `pty-${index}`) + let active = 0 + let peakActive = 0 + const attempts = new Map() + const mockAttach = vi.fn((id: string) => { + attempts.set(id, (attempts.get(id) ?? 0) + 1) + if (id === 'pty-0') { + return new Promise(() => {}) + } + if (id === 'pty-1') { + return Promise.reject(new Error('isolated attach failure')) + } + active++ + peakActive = Math.max(peakActive, active) + return new Promise((resolve) => { + setTimeout(() => { + active-- + resolve() + }, 100) + }) + }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue(ptyIds) + const random = vi.spyOn(Math, 'random').mockReturnValue(0) + vi.useFakeTimers() + + try { + const reconnect = session.reconnect(mockConn) + await vi.advanceTimersByTimeAsync(750) + + expect( + vi + .mocked(mockStore.markSshRemotePtyLease) + .mock.calls.filter(([, , state]) => state === 'attached') + .map(([, id]) => id) + ).toHaveLength(48) + expect(peakActive).toBeLessThanOrEqual(8) + + await vi.advanceTimersByTimeAsync(20_000) + await reconnect + + expect(mockAttach).toHaveBeenCalledTimes(52) + expect(attempts.get('pty-0')).toBe(2) + expect(attempts.get('pty-1')).toBe(2) + expect(session.getState()).toBe('ready') + expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + 'target-1', + expect.any(String), + 'expired' + ) + } finally { + vi.useRealTimers() + random.mockRestore() + } + }) + it('restores and persists exact incarnation proof from reconnect attach', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const incarnationId = 'incarnation-reconnect' diff --git a/src/main/ssh/ssh-relay-session-recovery-races.test.ts b/src/main/ssh/ssh-relay-session-recovery-races.test.ts new file mode 100644 index 00000000000..22b32ce3e35 --- /dev/null +++ b/src/main/ssh/ssh-relay-session-recovery-races.test.ts @@ -0,0 +1,361 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { SshRelaySession } from './ssh-relay-session' +import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' + +const { + acceptOutputDataMock, + muxRequestMock, + onNotificationByMethodMock, + openConsumerSessionMock, + muxDisposeMock, + attachForReconnectMock, + ptyDataHandlerRef +} = vi.hoisted(() => ({ + acceptOutputDataMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn(), + onNotificationByMethodMock: vi.fn(), + openConsumerSessionMock: vi.fn(), + muxDisposeMock: vi.fn(), + attachForReconnectMock: vi.fn().mockResolvedValue({}), + ptyDataHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) } +})) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: openConsumerSessionMock +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: acceptOutputDataMock, + acceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined), + allocateSshPtyProviderGeneration: vi.fn(() => 23), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}), + applySshPtySourceCancellationProof: vi.fn() +})) +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = muxRequestMock + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = onNotificationByMethodMock.mockImplementation(() => () => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = muxDisposeMock + isDisposed = vi.fn().mockReturnValue(false) + } +})) +vi.mock('../providers/ssh-pty-provider', () => ({ + isSshPtyNotFoundError: vi.fn().mockReturnValue(false), + isSshPtyIdentityMismatchError: vi.fn().mockReturnValue(false), + SshPtyProvider: class MockSshPtyProvider { + onData = vi.fn().mockImplementation((handler) => { + ptyDataHandlerRef.current = handler + return () => {} + }) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockReturnValue(() => {}) + attachForReconnect = attachForReconnectMock + setPtyDeliveryPauseAdapter = vi.fn() + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../ipc/pty', () => ({ + registerSshPtyProvider: vi.fn(), + unregisterSshPtyProvider: vi.fn(), + getSshPtyProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }), + getPtyIdsForConnection: vi.fn().mockReturnValue([]), + clearPtyOwnershipForConnection: vi.fn(), + clearProviderPtyState: vi.fn(), + deletePtyOwnership: vi.fn(), + restorePtyIncarnation: vi.fn(), + setPtyOwnership: vi.fn() +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { + clearProviderPtyState, + clearPtyOwnershipForConnection, + deletePtyOwnership, + getSshPtyProvider, + getPtyIdsForConnection, + registerSshPtyProvider, + setPtyOwnership +} = await import('../ipc/pty') +const { closeSshPtyOutputGeneration, getSshPtyAcceptedSourceCheckpoints } = + await import('../ipc/ssh-pty-output-intake-registry') + +describe('SshRelaySession recovery race fencing', () => { + beforeEach(() => { + vi.clearAllMocks() + ptyDataHandlerRef.current = undefined + attachForReconnectMock.mockResolvedValue({}) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([]) + muxRequestMock.mockResolvedValue([]) + mockDeploySuccess() + }) + + function emitSourceFrame(args: { + targetId: string + token: string + clientGeneration: number + ownerGeneration: number + sourceStartSu: number + sourceEndSu: number + }): void { + ptyDataHandlerRef.current?.({ + id: `ssh:${args.targetId}@@pty-1`, + data: 'late', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: args.sourceEndSu - args.sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `${args.token}:${args.sourceStartSu}:${args.sourceEndSu}`, + clientGeneration: args.clientGeneration, + ownerGeneration: args.ownerGeneration, + deliveryToken: args.token, + sourceStartSu: args.sourceStartSu, + sourceEndSu: args.sourceEndSu + } + }) + } + + async function prepareRecovery(targetId: string): Promise<{ + session: SshRelaySession + deps: ReturnType + }> { + let generation = 0 + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: ++generation, + ownerGeneration: generation, + ownerLease: `owner-lease-${generation}`, + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: `ssh:${targetId}@@pty-1`, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const deps = createMockDeps() + const session = new SshRelaySession( + targetId, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward, + undefined, + undefined, + () => true + ) + await session.establish(deps.mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue([`ssh:${targetId}@@pty-1`]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + return { session, deps } + } + + it('retains the empty recovery end as the first post-activation source anchor', async () => { + const targetId = 'empty-recovery-gap' + const { session, deps } = await prepareRecovery(targetId) + attachForReconnectMock.mockImplementation(async () => { + queueMicrotask(() => { + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }) + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + } + } + }) + await session.reconnect(deps.mockConn) + const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length + + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 5, + sourceEndSu: 8 + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount + 1) + expect(closeSshPtyOutputGeneration).toHaveBeenLastCalledWith( + 23, + 'ssh_source_frame_stale_or_non_contiguous' + ) + }) + + it('drops late frames from a token after its cancellation proof is validated', async () => { + const targetId = 'late-after-cancel' + muxRequestMock.mockImplementation(async (method) => + method === 'pty.cancelDelivery' ? { canceled: true, sentEndSu: 8, creditedEndSu: 4 } : [] + ) + const { session, deps } = await prepareRecovery(targetId) + attachForReconnectMock.mockImplementation(async () => { + queueMicrotask(() => { + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 5, + sourceEndSu: 8 + }) + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }) + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + } + } + }) + await session.reconnect(deps.mockConn) + const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length + + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 8, + sourceEndSu: 12 + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'detached') + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + }) + + it('keeps a stale overlapping recovery from canceling or mutating its replacement', async () => { + const targetId = 'overlapping-recovery' + const { session, deps } = await prepareRecovery(targetId) + attachForReconnectMock.mockImplementation(async () => { + const ownerGeneration = openConsumerSessionMock.mock.calls.length + if (ownerGeneration === 3) { + queueMicrotask(() => { + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 3, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'replacement-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }) + }) + } + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: ownerGeneration, + ownerGeneration, + ptyIncarnation: 'incarnation-1', + deliveryToken: ownerGeneration === 2 ? 'stale-token' : 'replacement-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + } + } + }) + + const staleReconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => expect(attachForReconnectMock).toHaveBeenCalledTimes(1)) + await Promise.resolve() + const replacementReconnect = session.reconnect(deps.mockConn) + await Promise.all([staleReconnect, replacementReconnect]) + + const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[2]) + expect(recoveryRequests).toHaveLength(2) + expect(recoveryRequests[1]).toMatchObject({ + status: 'checkpoint', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + }) + expect(muxRequestMock.mock.calls.filter(([method]) => method === 'pty.cancelDelivery')).toEqual( + [] + ) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledTimes(1) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'attached') + expect(setPtyOwnership).toHaveBeenCalledTimes(1) + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + expect(session.getState()).toBe('ready') + }) +}) diff --git a/src/main/ssh/ssh-relay-session-terminal-error.test.ts b/src/main/ssh/ssh-relay-session-terminal-error.test.ts index 6b982fd1397..52be21dcc2e 100644 --- a/src/main/ssh/ssh-relay-session-terminal-error.test.ts +++ b/src/main/ssh/ssh-relay-session-terminal-error.test.ts @@ -30,6 +30,7 @@ vi.mock('./ssh-channel-multiplexer', () => { notify = vi.fn() request = vi.fn().mockResolvedValue([]) onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) onRequest = vi.fn().mockReturnValue(() => {}) onDispose = vi.fn().mockReturnValue(() => {}) dispose = vi.fn() diff --git a/src/main/ssh/ssh-relay-session.test.ts b/src/main/ssh/ssh-relay-session.test.ts index f6c9c80fc03..ac97b1e394a 100644 --- a/src/main/ssh/ssh-relay-session.test.ts +++ b/src/main/ssh/ssh-relay-session.test.ts @@ -725,7 +725,7 @@ describe('SshRelaySession', () => { }) }) - it('routes transient reattach failures through relay-lost retry handling', async () => { + it('retries transient reattach failure without tearing down provider registration', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) const onRelayLost = vi.fn() @@ -741,11 +741,17 @@ describe('SshRelaySession', () => { dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-live']) + const random = vi.spyOn(Math, 'random').mockReturnValue(0) - await session.reconnect(mockConn) + try { + await session.reconnect(mockConn) + } finally { + random.mockRestore() + } - expect(mockAttach).toHaveBeenCalledWith('pty-live') - expect(onRelayLost).toHaveBeenCalledWith('target-1') + expect(mockAttach).toHaveBeenCalledTimes(2) + expect(onRelayLost).not.toHaveBeenCalled() + expect(session.getState()).toBe('ready') expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( 'target-1', 'pty-live', diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index cea22a190de..2aa2faba9ff 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -42,9 +42,11 @@ import { acceptSshPtyOutputData, acceptSshPtyOutputExit, allocateSshPtyProviderGeneration, + applySshPtySourceCancellationProof, closeSshPtyOutputGeneration, getSshPtyAcceptedSourceCheckpoints, - installSshPtySourceAckPublisher + installSshPtySourceAckPublisher, + installSshPtySourceCancellationPublisher } from '../ipc/ssh-pty-output-intake-registry' import type { SshPtyAcceptedSourceCheckpoint } from '../ipc/ssh-pty-output-source-obligations' import { @@ -71,6 +73,7 @@ import { import type { Store } from '../persistence' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session' import { runRemoteOrcaCli } from './ssh-remote-orca-cli' import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' @@ -85,25 +88,31 @@ import type { PtySourceRecoveryPending, PtySourceRecoveryRequest } from '../../shared/pty-source-recovery-contract' +import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget' export type RelaySessionState = 'idle' | 'deploying' | 'ready' | 'reconnecting' | 'disposed' type SshPtyExitPayload = Parameters[0] type SshPtyDataPayload = Parameters[0] -const SSH_PTY_RECOVERY_QUARANTINE_MAX_BYTES = DEFAULT_PTY_SOURCE_WINDOW_SU * 8 -const SSH_PTY_RECOVERY_QUARANTINE_MAX_FRAMES = 1_024 +type SshPtyLease = ReturnType[number] +const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8 +const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000 +const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50 +const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200 type PendingPtyReattach = { + mux: SshChannelMultiplexer + providerGeneration: number + retentionKey: string exits: SshPtyExitPayload[] queuedData: SshPtyDataPayload[] + recoveryData: SshPtyDataPayload[] liveData: SshPtyDataPayload[] recovery?: PtySourceRecoveryPending recoveryComplete?: PtySourceRecoveryComplete + nextRecoverySourceSu?: number replacementDeliveryToken?: string restoreRequired?: string recoveryWaiters: Set<() => void> - recoveryAdmissions: Promise[] - queuedBytes: number - queuedFrames: number livePassthrough: boolean activated: boolean } @@ -236,8 +245,10 @@ export class SshRelaySession { private hostPlatform: RemoteHostPlatform | null = null private remoteCliBridgeEnv: RemoteCliBridgeEnv | null = null private pendingPtyReattaches = new Map() + private readonly ptyRecoveryRetention = new SshPtyRecoveryRetentionBudget() private activePtyProviderGeneration: number | null = null private sourceAckPublisherCleanup: (() => void) | null = null + private sourceCancellationPublisherCleanup: (() => void) | null = null private ptyRecoveryNotificationCleanups: (() => void)[] = [] private readonly sourceIdentityByRelayPtyId = new Map< string, @@ -246,8 +257,10 @@ export class SshRelaySession { clientGeneration: number ownerGeneration: number ptyIncarnation: string + nextSourceSu?: number }> >() + private readonly blockedSourceDeliveryKeys = new Set() private readonly ptyConsumerClientInstanceId: string private ptyConsumerSessionState: SshPtyConsumerSessionState | null = null @@ -393,23 +406,7 @@ export class SshRelaySession { this.mux = mux const ownsAttempt = (): boolean => this.mux === mux && !this.isDisposed() - const previousOwner = this.negotiatedPtyConsumerOwner(serverBuildId) - this.ptyConsumerSessionState = await openSshPtyConsumerSession(mux, { - clientInstanceId: this.ptyConsumerClientInstanceId, - expectedServerBuildId: serverBuildId, - allowSameBuildLegacyFallback: true, - ...(this.isPtySourceCreditEnabled() - ? { outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } } - : {}), - ...(previousOwner - ? { - resume: { - ownerGeneration: previousOwner.ownerGeneration, - ownerLease: previousOwner.ownerLease - } - } - : {}) - }) + this.ptyConsumerSessionState = await this.openPtyConsumerSession(mux, serverBuildId) this.rememberPtyConsumerRecovery(serverBuildId) await mux.request('session.resolveHome', { path: '~' }) @@ -433,7 +430,7 @@ export class SshRelaySession { } // Why: explicit disconnect keeps PTY ownership, so a later manual connect must reattach those remote PTYs. - await this.reattachKnownPtys(ownsAttempt) + await this.reattachKnownPtys(mux, ownsAttempt) if (!ownsAttempt()) { throw new Error('Session disposed during establish') @@ -529,23 +526,7 @@ export class SshRelaySession { !abortController.signal.aborted && !this.isDisposed() - const previousOwner = this.negotiatedPtyConsumerOwner(serverBuildId) - this.ptyConsumerSessionState = await openSshPtyConsumerSession(mux, { - clientInstanceId: this.ptyConsumerClientInstanceId, - expectedServerBuildId: serverBuildId, - allowSameBuildLegacyFallback: true, - ...(this.isPtySourceCreditEnabled() - ? { outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } } - : {}), - ...(previousOwner - ? { - resume: { - ownerGeneration: previousOwner.ownerGeneration, - ownerLease: previousOwner.ownerLease - } - } - : {}) - }) + this.ptyConsumerSessionState = await this.openPtyConsumerSession(mux, serverBuildId) this.rememberPtyConsumerRecovery(serverBuildId) if (!ownsAttempt()) { if (!mux.isDisposed()) { @@ -584,7 +565,7 @@ export class SshRelaySession { return } - await this.reattachKnownPtys(ownsAttempt) + await this.reattachKnownPtys(mux, ownsAttempt) if (!ownsAttempt()) { return @@ -743,6 +724,8 @@ export class SshRelaySession { } this.sourceAckPublisherCleanup?.() this.sourceAckPublisherCleanup = null + this.sourceCancellationPublisherCleanup?.() + this.sourceCancellationPublisherCleanup = null if (consumerOwnerState?.outputFlowControl) { this.sourceAckPublisherCleanup = installSshPtySourceAckPublisher( providerGeneration, @@ -753,6 +736,26 @@ export class SshRelaySession { onSettled ) ) + this.sourceCancellationPublisherCleanup = installSshPtySourceCancellationPublisher( + providerGeneration, + async (request) => { + const result = (await mux.request('pty.cancelDelivery', { + ...request, + id: toRelaySshPtyId(this.targetId, request.id) + })) as Record + if ( + result.canceled !== true || + !Number.isSafeInteger(result.sentEndSu) || + !Number.isSafeInteger(result.creditedEndSu) + ) { + throw new Error('ssh_source_cancellation_proof_invalid') + } + return { + sentEndSu: result.sentEndSu as number, + creditedEndSu: result.creditedEndSu as number + } + } + ) } this.activePtyProviderGeneration = providerGeneration registerSshPtyProvider(this.targetId, ptyProvider) @@ -796,7 +799,7 @@ export class SshRelaySession { ) registerSshGitProvider(this.targetId, gitProvider) - this.wireUpPtyEvents(ptyProvider) + this.wireUpPtyEvents(ptyProvider, mux, providerGeneration) this.wireUpAgentHookEvents(mux) this.wireUpRemoteWorkspaceEvents(mux) return true @@ -813,6 +816,48 @@ export class SshRelaySession { : null } + private async openPtyConsumerSession( + mux: SshChannelMultiplexer, + serverBuildId: string | undefined + ): Promise { + const previousOwner = this.negotiatedPtyConsumerOwner(serverBuildId) + const options = { + clientInstanceId: this.ptyConsumerClientInstanceId, + expectedServerBuildId: serverBuildId, + allowSameBuildLegacyFallback: true, + ...(this.isPtySourceCreditEnabled() + ? { outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } } + : {}) + } + try { + return await openSshPtyConsumerSession(mux, { + ...options, + ...(previousOwner + ? { + resume: { + ownerGeneration: previousOwner.ownerGeneration, + ownerLease: previousOwner.ownerLease + } + } + : {}) + }) + } catch (error) { + if ( + !previousOwner || + (error as { code?: unknown }).code !== PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR + ) { + throw error + } + const recovery = ptyConsumerRecoveryByTarget.get(this.targetId) + if (recovery) { + delete recovery.owner + recovery.checkpointsByAppPtyId.clear() + } + this.ptyConsumerSessionState = null + return openSshPtyConsumerSession(mux, options) + } + } + private rememberPtyConsumerRecovery(serverBuildId: string | undefined): void { const owner = this.negotiatedPtyConsumerOwner() if (!owner || !serverBuildId) { @@ -1092,6 +1137,8 @@ export class SshRelaySession { } this.sourceAckPublisherCleanup?.() this.sourceAckPublisherCleanup = null + this.sourceCancellationPublisherCleanup?.() + this.sourceCancellationPublisherCleanup = null if (this.mux && !this.mux.isDisposed()) { this.mux.dispose(reason) } @@ -1117,12 +1164,14 @@ export class SshRelaySession { unregisterSshFilesystemProvider(this.targetId) unregisterSshGitProvider(this.targetId) this.sourceIdentityByRelayPtyId.clear() + this.blockedSourceDeliveryKeys.clear() for (const pending of this.pendingPtyReattaches.values()) { for (const resolve of pending.recoveryWaiters) { resolve() } } this.pendingPtyReattaches.clear() + this.ptyRecoveryRetention.clear() } // Why: back-compat for old relays that gate FS ops on registered roots; removable post-cutover (docs/relay-fs-allowlist-removal.md). @@ -1194,8 +1243,19 @@ export class SshRelaySession { } } - private wireUpPtyEvents(ptyProvider: SshPtyProvider): void { + private wireUpPtyEvents( + ptyProvider: SshPtyProvider, + mux: SshChannelMultiplexer, + providerGeneration: number + ): void { ptyProvider.onData((payload) => { + if ( + this.mux !== mux || + this.activePtyProviderGeneration !== providerGeneration || + payload.providerGeneration !== providerGeneration + ) { + return + } const pending = this.pendingPtyReattaches.get(payload.id) if (pending && this.negotiatedPtyConsumerOwner()?.outputFlowControl) { if (pending.livePassthrough) { @@ -1208,12 +1268,22 @@ export class SshRelaySession { void this.acceptPtyData(payload).catch(() => {}) }) ptyProvider.onReplay((payload) => { + if (this.mux !== mux || this.activePtyProviderGeneration !== providerGeneration) { + return + } const win = this.getMainWindow() if (win && !win.isDestroyed()) { win.webContents.send('pty:replay', payload) } }) ptyProvider.onExit((payload) => { + if ( + this.mux !== mux || + this.activePtyProviderGeneration !== providerGeneration || + payload.providerGeneration !== providerGeneration + ) { + return + } const pendingReattach = this.pendingPtyReattaches.get(payload.id) if (pendingReattach && !pendingReattach.activated) { // Why: attach response and exit can share one transport batch, before incarnation restoration runs. @@ -1229,12 +1299,21 @@ export class SshRelaySession { private acceptPtyData(payload: SshPtyDataPayload): Promise { const consumerOwner = this.negotiatedPtyConsumerOwner() + const offeredSource = payload.source + if ( + offeredSource && + this.blockedSourceDeliveryKeys.has( + this.sourceDeliveryKey(payload.providerGeneration, offeredSource) + ) + ) { + return Promise.resolve() + } if ( consumerOwner?.outputFlowControl && - (!payload.source || + (!offeredSource || payload.sourceMalformed || - payload.source.clientGeneration !== consumerOwner.clientGeneration || - payload.source.ownerGeneration !== consumerOwner.ownerGeneration) + offeredSource.clientGeneration !== consumerOwner.clientGeneration || + offeredSource.ownerGeneration !== consumerOwner.ownerGeneration) ) { closeSshPtyOutputGeneration( payload.providerGeneration, @@ -1243,13 +1322,31 @@ export class SshRelaySession { this.mux?.dispose('connection_lost') return Promise.reject(new Error('ssh_source_frame_malformed_or_missing')) } - const source = consumerOwner?.outputFlowControl ? payload.source : undefined + const source = consumerOwner?.outputFlowControl ? offeredSource : undefined if (source) { + const current = this.sourceIdentityByRelayPtyId.get(source.relayPtyId) + if ( + source.sourceEndSu <= source.sourceStartSu || + (current && + (current.deliveryToken !== source.deliveryToken || + current.clientGeneration !== source.clientGeneration || + current.ownerGeneration !== source.ownerGeneration || + current.ptyIncarnation !== payload.ptyIncarnation || + (current.nextSourceSu !== undefined && current.nextSourceSu !== source.sourceStartSu))) + ) { + closeSshPtyOutputGeneration( + payload.providerGeneration, + 'ssh_source_frame_stale_or_non_contiguous' + ) + this.mux?.dispose('connection_lost') + return Promise.reject(new Error('ssh_source_frame_stale_or_non_contiguous')) + } this.sourceIdentityByRelayPtyId.set(source.relayPtyId, { deliveryToken: source.deliveryToken, clientGeneration: source.clientGeneration, ownerGeneration: source.ownerGeneration, - ptyIncarnation: payload.ptyIncarnation + ptyIncarnation: payload.ptyIncarnation, + nextSourceSu: source.sourceEndSu }) } const rawLength = payload.sequenceChars ?? payload.data.length @@ -1266,12 +1363,13 @@ export class SshRelaySession { } private quarantineReattachData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void { - pending.queuedBytes += Buffer.byteLength(payload.data, 'utf8') - pending.queuedFrames++ - if ( - pending.queuedBytes > SSH_PTY_RECOVERY_QUARANTINE_MAX_BYTES || - pending.queuedFrames > SSH_PTY_RECOVERY_QUARANTINE_MAX_FRAMES - ) { + if (pending.restoreRequired) { + return + } + const sourceSu = payload.source + ? payload.source.sourceEndSu - payload.source.sourceStartSu + : (payload.sequenceChars ?? payload.data.length) + if (!this.ptyRecoveryRetention.tryRetain(pending.retentionKey, payload.data, sourceSu)) { pending.restoreRequired = 'recoveryQuarantineCapacityExceeded' this.wakeRecovery(pending) return @@ -1288,13 +1386,20 @@ export class SshRelaySession { } private admitRecoveryData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void { + if (pending.restoreRequired) { + return + } const recovery = pending.recovery + const nextSourceSu = pending.nextRecoverySourceSu if ( !recovery || !payload.source || + nextSourceSu === undefined || payload.source.deliveryToken !== recovery.deliveryToken || payload.source.clientGeneration !== recovery.clientGeneration || payload.source.ownerGeneration !== recovery.ownerGeneration || + payload.source.sourceStartSu !== nextSourceSu || + payload.source.sourceEndSu <= payload.source.sourceStartSu || payload.source.sourceEndSu > recovery.recoveryEndSu || payload.ptyIncarnation !== recovery.ptyIncarnation ) { @@ -1302,13 +1407,8 @@ export class SshRelaySession { this.wakeRecovery(pending) return } - const admission = this.acceptPtyData(payload).catch((error) => { - pending.restoreRequired = - error instanceof Error ? error.message : 'recoveryIntakeAdmissionFailed' - this.wakeRecovery(pending) - throw error - }) - pending.recoveryAdmissions.push(admission) + pending.nextRecoverySourceSu = payload.source.sourceEndSu + pending.recoveryData.push(payload) } private installPtyRecoveryNotifications(mux: SshChannelMultiplexer): void { @@ -1317,9 +1417,12 @@ export class SshRelaySession { } this.ptyRecoveryNotificationCleanups = [ mux.onNotificationByMethod('pty.recoveryComplete', (params) => { + if (this.mux !== mux) { + return + } const id = typeof params.id === 'string' ? toAppSshPtyId(this.targetId, params.id) : '' const pending = this.pendingPtyReattaches.get(id) - if (!pending) { + if (!pending || pending.mux !== mux) { return } const complete = parseRecoveryComplete(params) @@ -1331,9 +1434,12 @@ export class SshRelaySession { this.wakeRecovery(pending) }), mux.onNotificationByMethod('pty.restoreRequired', (params) => { + if (this.mux !== mux) { + return + } const id = typeof params.id === 'string' ? toAppSshPtyId(this.targetId, params.id) : '' const pending = this.pendingPtyReattaches.get(id) - if (!pending) { + if (!pending || pending.mux !== mux) { return } pending.restoreRequired = @@ -1341,6 +1447,9 @@ export class SshRelaySession { this.wakeRecovery(pending) }), mux.onNotificationByMethod('pty.deliveryCanceled', (params) => { + if (this.mux !== mux) { + return + } const id = typeof params.id === 'string' ? params.id : '' const identity = this.sourceIdentityByRelayPtyId.get(id) if ( @@ -1355,7 +1464,7 @@ export class SshRelaySession { const replacementDeliveryToken = typeof params.replacementDeliveryToken === 'string' ? params.replacementDeliveryToken : '' const pending = this.pendingPtyReattaches.get(toAppSshPtyId(this.targetId, id)) - if (pending) { + if (pending?.mux === mux) { if ( replacementDeliveryToken.length === 0 || replacementDeliveryToken === identity.deliveryToken @@ -1379,10 +1488,35 @@ export class SshRelaySession { return } const generation = this.activePtyProviderGeneration - if (generation !== null) { - closeSshPtyOutputGeneration(generation, 'ssh_source_delivery_canceled') + if ( + generation !== null && + Number.isSafeInteger(params.sentEndSu) && + Number.isSafeInteger(params.creditedEndSu) + ) { + try { + applySshPtySourceCancellationProof( + { + id: toAppSshPtyId(this.targetId, id), + code: -1, + providerGeneration: generation, + ptyIncarnation: identity.ptyIncarnation + }, + { + sentEndSu: params.sentEndSu as number, + creditedEndSu: params.creditedEndSu as number + } + ) + this.blockedSourceDeliveryKeys.add( + this.sourceDeliveryKey(generation, { + relayPtyId: id, + ...identity + }) + ) + this.sourceIdentityByRelayPtyId.delete(id) + } catch { + /* Invalid proof retains the active token identity. */ + } } - mux.dispose('connection_lost') }) ] } @@ -1435,7 +1569,10 @@ export class SshRelaySession { } } - private async reattachKnownPtys(shouldContinue: () => boolean): Promise { + private async reattachKnownPtys( + mux: SshChannelMultiplexer, + shouldContinue: () => boolean + ): Promise { const activeLeases = this.store .getSshRemotePtyLeases(this.targetId) .filter((lease) => lease.state !== 'terminated' && lease.state !== 'expired') @@ -1460,142 +1597,285 @@ export class SshRelaySession { ]) ) const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined - if (!ptyProvider) { + const providerGeneration = this.activePtyProviderGeneration + if (!ptyProvider || providerGeneration === null || this.mux !== mux) { return } - for (const ptyId of ptyIds) { + let nextPtyIndex = 0 + const worker = async (): Promise => { + while (shouldContinue()) { + const ptyId = ptyIds[nextPtyIndex++] + if (ptyId === undefined) { + return + } + try { + await this.reattachKnownPty({ + ptyProvider, + ptyId, + activeLeaseByPtyId, + expectedIdentityByPtyId, + mux, + providerGeneration, + shouldContinue + }) + } catch (error) { + console.warn( + `[ssh-relay-session] PTY ${ptyId} reattach processing failed for ${this.targetId}: ${ + error instanceof Error ? error.message : String(error) + }` + ) + } + } + } + await Promise.all( + Array.from({ length: Math.min(SSH_PTY_REATTACH_MAX_CONCURRENCY, ptyIds.length) }, worker) + ) + } + + private async reattachKnownPty(args: { + ptyProvider: SshPtyProvider + ptyId: string + activeLeaseByPtyId: Map + expectedIdentityByPtyId: Map + mux: SshChannelMultiplexer + providerGeneration: number + shouldContinue: () => boolean + }): Promise { + const { + ptyProvider, + ptyId, + activeLeaseByPtyId, + expectedIdentityByPtyId, + mux, + providerGeneration, + shouldContinue + } = args + const appPtyId = toAppSshPtyId(this.targetId, ptyId) + const pendingReattach: PendingPtyReattach = { + mux, + providerGeneration, + retentionKey: `${providerGeneration}\0${appPtyId}\0${randomUUID()}`, + exits: [], + queuedData: [], + recoveryData: [], + liveData: [], + recoveryWaiters: new Set(), + livePassthrough: false, + activated: false + } + this.pendingPtyReattaches.set(appPtyId, pendingReattach) + try { + const recoveryRequest = this.sourceRecoveryRequest(appPtyId) + const attachResult = await this.attachPtyWithRetry( + ptyProvider, + ptyId, + expectedIdentityByPtyId.get(ptyId), + recoveryRequest, + shouldContinue + ) if (!shouldContinue()) { return } - const appPtyId = toAppSshPtyId(this.targetId, ptyId) - const pendingReattach: PendingPtyReattach = { - exits: [], - queuedData: [], - liveData: [], - recoveryWaiters: new Set(), - recoveryAdmissions: [], - queuedBytes: 0, - queuedFrames: 0, - livePassthrough: false, - activated: false - } - this.pendingPtyReattaches.set(appPtyId, pendingReattach) - try { - const expectedIdentity = expectedIdentityByPtyId.get(ptyId) - const recoveryRequest = this.sourceRecoveryRequest(appPtyId) - const attachResult = - (expectedIdentity - ? recoveryRequest - ? await ptyProvider.attachForReconnect(ptyId, expectedIdentity, recoveryRequest) - : await ptyProvider.attachForReconnect(ptyId, expectedIdentity) - : recoveryRequest - ? await ptyProvider.attachForReconnect(ptyId, undefined, recoveryRequest) - : await ptyProvider.attachForReconnect(ptyId)) ?? {} - if (!shouldContinue()) { - return - } - const exitDuringAttach = pendingReattach.exits.find( - (exit) => - !exit.incarnationId || - !attachResult.incarnationId || - exit.incarnationId === attachResult.incarnationId - ) - if (exitDuringAttach) { - if (attachResult.incarnationId) { - restorePtyIncarnation(appPtyId, attachResult.incarnationId) - this.runtime?.acceptPtyIncarnationForExit(appPtyId, attachResult.incarnationId) - } - await this.acceptPtyExit(exitDuringAttach) - continue - } - if ( - recoveryRequest && - !(await this.finishSourceRecovery( - ptyProvider, - ptyId, - appPtyId, - attachResult, - recoveryRequest, - pendingReattach, - shouldContinue - )) - ) { - continue - } - setPtyOwnership(appPtyId, this.targetId) + const exitDuringAttach = pendingReattach.exits.find( + (exit) => + !exit.incarnationId || + !attachResult.incarnationId || + exit.incarnationId === attachResult.incarnationId + ) + if (exitDuringAttach) { if (attachResult.incarnationId) { restorePtyIncarnation(appPtyId, attachResult.incarnationId) - const lease = activeLeaseByPtyId.get(ptyId) - if (lease?.worktreeId && lease.tabId && lease.leafId) { - this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { - tabId: lease.tabId, - leafId: lease.leafId, - incarnationId: attachResult.incarnationId - }) - // Why: reconnect may be the first new-relay response that can backfill exact exit fencing. - try { - this.store.persistPtyBinding({ - worktreeId: lease.worktreeId, - tabId: lease.tabId, - leafId: lease.leafId, - ptyId: appPtyId, - incarnationId: attachResult.incarnationId - }) - } catch (error) { - // Why: this backfill improves future fencing but must not disconnect an already-live relay PTY. - console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error) - } - } else { - this.runtime?.onPtySpawned(appPtyId, attachResult.incarnationId, { - awaitsRegistration: false - }) - } - } - this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached') - pendingReattach.activated = true - const exitAfterActivation = pendingReattach.exits.find( - (exit) => - !exit.incarnationId || - !attachResult.incarnationId || - exit.incarnationId === attachResult.incarnationId - ) - if (exitAfterActivation) { - await this.acceptPtyExit(exitAfterActivation) - continue - } - if (!recoveryRequest) { - this.forwardReattachReplay(appPtyId, attachResult.replay ?? '') - } - } catch (err) { - if (!isSshPtyNotFoundError(err)) { - throw err - } - if (isSshPtyIdentityMismatchError(err)) { - console.warn( - `[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${ - err instanceof Error ? err.message : String(err) - }` - ) - continue - } - console.warn( - `[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${ - err instanceof Error ? err.message : String(err) - }` - ) - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') - // Why: reattach failure means the remote process is gone; tell the renderer to clear the stale pane. - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) - } - } finally { - if (this.pendingPtyReattaches.get(appPtyId) === pendingReattach) { - this.pendingPtyReattaches.delete(appPtyId) + this.runtime?.acceptPtyIncarnationForExit(appPtyId, attachResult.incarnationId) } + await this.acceptPtyExit(exitDuringAttach) + return } + if ( + recoveryRequest && + !(await this.finishSourceRecovery( + ptyId, + appPtyId, + attachResult, + recoveryRequest, + pendingReattach, + shouldContinue + )) + ) { + return + } + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pendingReattach)) { + return + } + setPtyOwnership(appPtyId, this.targetId) + if (attachResult.incarnationId) { + restorePtyIncarnation(appPtyId, attachResult.incarnationId) + this.restoreReattachedPtyRuntime( + appPtyId, + attachResult.incarnationId, + activeLeaseByPtyId.get(ptyId) + ) + } + this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached') + pendingReattach.activated = true + const exitAfterActivation = pendingReattach.exits.find( + (exit) => + !exit.incarnationId || + !attachResult.incarnationId || + exit.incarnationId === attachResult.incarnationId + ) + if (exitAfterActivation) { + await this.acceptPtyExit(exitAfterActivation) + return + } + if (!recoveryRequest) { + this.forwardReattachReplay(appPtyId, attachResult.replay ?? '') + } + } catch (error) { + if (!shouldContinue()) { + return + } + this.handlePtyReattachFailure(ptyId, appPtyId, pendingReattach, error) + } finally { + if (this.pendingPtyReattaches.get(appPtyId) === pendingReattach) { + this.pendingPtyReattaches.delete(appPtyId) + } + this.ptyRecoveryRetention.release(pendingReattach.retentionKey) + } + } + + private restoreReattachedPtyRuntime( + appPtyId: string, + incarnationId: string, + lease: SshPtyLease | undefined + ): void { + if (lease?.worktreeId && lease.tabId && lease.leafId) { + this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { + tabId: lease.tabId, + leafId: lease.leafId, + incarnationId + }) + try { + this.store.persistPtyBinding({ + worktreeId: lease.worktreeId, + tabId: lease.tabId, + leafId: lease.leafId, + ptyId: appPtyId, + incarnationId + }) + } catch (error) { + console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error) + } + return + } + this.runtime?.onPtySpawned(appPtyId, incarnationId, { awaitsRegistration: false }) + } + + private async attachPtyWithRetry( + ptyProvider: SshPtyProvider, + ptyId: string, + expectedIdentity: ExpectedPtyIdentity | undefined, + recoveryRequest: PtySourceRecoveryRequest | undefined, + shouldContinue: () => boolean + ): Promise { + let lastError: unknown + for (let attempt = 0; attempt < 2; attempt++) { + if (!shouldContinue()) { + throw lastError ?? new Error('PTY reattach attempt is no longer current') + } + try { + return await this.attachPtyWithDeadline( + ptyProvider, + ptyId, + expectedIdentity, + recoveryRequest + ) + } catch (error) { + lastError = error + if (!shouldContinue() || isSshPtyNotFoundError(error) || attempt === 1) { + throw error + } + await this.waitForPtyReattachRetry() + } + } + throw lastError + } + + private async attachPtyWithDeadline( + ptyProvider: SshPtyProvider, + ptyId: string, + expectedIdentity: ExpectedPtyIdentity | undefined, + recoveryRequest: PtySourceRecoveryRequest | undefined + ): Promise { + let timer: ReturnType | undefined + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + reject( + new Error(`PTY reattach attempt timed out after ${SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS}ms`) + ) + }, SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS) + timer.unref?.() + }) + try { + const attach = expectedIdentity + ? recoveryRequest + ? ptyProvider.attachForReconnect(ptyId, expectedIdentity, recoveryRequest) + : ptyProvider.attachForReconnect(ptyId, expectedIdentity) + : recoveryRequest + ? ptyProvider.attachForReconnect(ptyId, undefined, recoveryRequest) + : ptyProvider.attachForReconnect(ptyId) + return (await Promise.race([attach, timeout])) ?? {} + } finally { + if (timer) { + clearTimeout(timer) + } + } + } + + private async waitForPtyReattachRetry(): Promise { + const delayMs = + SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS + + Math.floor(Math.random() * (SSH_PTY_REATTACH_RETRY_JITTER_MS + 1)) + await new Promise((resolve) => { + const timer = setTimeout(resolve, delayMs) + timer.unref?.() + }) + } + + private handlePtyReattachFailure( + ptyId: string, + appPtyId: string, + pending: PendingPtyReattach, + error: unknown + ): void { + if (!isSshPtyNotFoundError(error)) { + pending.restoreRequired = 'reattachAttemptsExhausted' + this.wakeRecovery(pending) + console.warn( + `[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after bounded reattach attempts failed: ${ + error instanceof Error ? error.message : String(error) + }` + ) + return + } + if (isSshPtyIdentityMismatchError(error)) { + console.warn( + `[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${ + error instanceof Error ? error.message : String(error) + }` + ) + return + } + console.warn( + `[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${ + error instanceof Error ? error.message : String(error) + }` + ) + clearProviderPtyState(appPtyId) + deletePtyOwnership(appPtyId) + this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') + const win = this.getMainWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) } } @@ -1620,7 +1900,6 @@ export class SshRelaySession { } private async finishSourceRecovery( - ptyProvider: SshPtyProvider, relayPtyId: string, appPtyId: string, attachResult: SshPtyAttachResult, @@ -1642,15 +1921,20 @@ export class SshRelaySession { (pending.replacementDeliveryToken !== undefined && pending.replacementDeliveryToken !== recovery.deliveryToken) ) { - await this.resetPtyAfterRecoveryFailure(ptyProvider, relayPtyId, appPtyId) + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) return false } pending.recovery = recovery + pending.nextRecoverySourceSu = recovery.checkpointSourceEndSu this.sourceIdentityByRelayPtyId.set(relayPtyId, { deliveryToken: recovery.deliveryToken, clientGeneration: recovery.clientGeneration, ownerGeneration: recovery.ownerGeneration, - ptyIncarnation: recovery.ptyIncarnation + ptyIncarnation: recovery.ptyIncarnation, + nextSourceSu: recovery.checkpointSourceEndSu }) for (const payload of pending.queuedData.splice(0)) { this.admitRecoveryData(pending, payload) @@ -1666,19 +1950,50 @@ export class SshRelaySession { complete.ownerGeneration !== recovery.ownerGeneration || complete.ptyIncarnation !== recovery.ptyIncarnation || complete.checkpointSourceEndSu !== recovery.checkpointSourceEndSu || - complete.recoveryEndSu !== recovery.recoveryEndSu + complete.recoveryEndSu !== recovery.recoveryEndSu || + pending.nextRecoverySourceSu !== recovery.recoveryEndSu ) { - await this.resetPtyAfterRecoveryFailure(ptyProvider, relayPtyId, appPtyId) + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) return false } + let nextLiveSourceSu = recovery.recoveryEndSu + for (const payload of pending.liveData) { + if ( + !payload.source || + payload.source.deliveryToken !== recovery.deliveryToken || + payload.source.clientGeneration !== recovery.clientGeneration || + payload.source.ownerGeneration !== recovery.ownerGeneration || + payload.source.sourceStartSu !== nextLiveSourceSu || + payload.source.sourceEndSu <= payload.source.sourceStartSu || + payload.ptyIncarnation !== recovery.ptyIncarnation + ) { + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) + return false + } + nextLiveSourceSu = payload.source.sourceEndSu + } try { - await Promise.all(pending.recoveryAdmissions) + for (const payload of pending.recoveryData) { + await this.acceptPtyData(payload) + } for (const payload of pending.liveData) { await this.acceptPtyData(payload) } pending.livePassthrough = true } catch { - await this.resetPtyAfterRecoveryFailure(ptyProvider, relayPtyId, appPtyId) + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) + return false + } + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { return false } const acceptedSourceEndSu = pending.liveData.reduce( @@ -1704,7 +2019,7 @@ export class SshRelaySession { pending: PendingPtyReattach, shouldContinue: () => boolean ): Promise { - const deadline = Date.now() + 30_000 + const deadline = Date.now() + SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS while ( shouldContinue() && !pending.recoveryComplete && @@ -1732,25 +2047,125 @@ export class SshRelaySession { } } - private async resetPtyAfterRecoveryFailure( - ptyProvider: SshPtyProvider, + private async abandonPtySourceRecovery( relayPtyId: string, - appPtyId: string + appPtyId: string, + pending: PendingPtyReattach ): Promise { - try { - await ptyProvider.shutdown(appPtyId, { immediate: true, deadlineMs: Date.now() + 5_000 }) - } catch { - /* The recovery fence already prevents stale output activation. */ + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + const recovery = pending.recovery + const { mux, providerGeneration } = pending + if (recovery && !mux.isDisposed()) { + const cancellationRequest = { + id: relayPtyId, + clientGeneration: recovery.clientGeneration, + ownerGeneration: recovery.ownerGeneration, + deliveryToken: recovery.deliveryToken + } + this.blockedSourceDeliveryKeys.add( + this.sourceDeliveryKey(providerGeneration, { + relayPtyId, + deliveryToken: recovery.deliveryToken, + clientGeneration: recovery.clientGeneration, + ownerGeneration: recovery.ownerGeneration + }) + ) + try { + const result = (await mux.request('pty.cancelDelivery', cancellationRequest)) as Record< + string, + unknown + > + if ( + result.canceled !== true || + !Number.isSafeInteger(result.sentEndSu) || + !Number.isSafeInteger(result.creditedEndSu) + ) { + throw new Error('ssh_source_cancellation_proof_invalid') + } + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + const identity = this.sourceIdentityByRelayPtyId.get(relayPtyId) + if (identity && !this.sameSourceDelivery(identity, recovery)) { + return + } + if (identity) { + applySshPtySourceCancellationProof( + { + id: appPtyId, + code: -1, + providerGeneration, + ptyIncarnation: recovery.ptyIncarnation + }, + { + sentEndSu: result.sentEndSu as number, + creditedEndSu: result.creditedEndSu as number + } + ) + } + } catch (error) { + console.warn( + `[ssh-relay-session] Failed to cancel replacement delivery for ${relayPtyId}: ${ + error instanceof Error ? error.message : String(error) + }` + ) + return + } + } + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + const identity = this.sourceIdentityByRelayPtyId.get(relayPtyId) + if (!identity || !recovery || this.sameSourceDelivery(identity, recovery)) { + this.sourceIdentityByRelayPtyId.delete(relayPtyId) } - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - this.sourceIdentityByRelayPtyId.delete(relayPtyId) ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.delete(appPtyId) ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.delete(relayPtyId) - this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'expired') - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) - } + this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'detached') + } + + private ownsPtyRecoveryAttempt(appPtyId: string, pending: PendingPtyReattach): boolean { + return ( + this.pendingPtyReattaches.get(appPtyId) === pending && + this.mux === pending.mux && + this.activePtyProviderGeneration === pending.providerGeneration && + !pending.mux.isDisposed() + ) + } + + private sameSourceDelivery( + left: Readonly<{ + deliveryToken: string + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + }>, + right: Readonly<{ + deliveryToken: string + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + }> + ): boolean { + return ( + left.deliveryToken === right.deliveryToken && + left.clientGeneration === right.clientGeneration && + left.ownerGeneration === right.ownerGeneration && + left.ptyIncarnation === right.ptyIncarnation + ) + } + + private sourceDeliveryKey( + providerGeneration: number, + source: Readonly<{ + relayPtyId?: string + deliveryToken: string + clientGeneration: number + ownerGeneration: number + }> + ): string { + return `${providerGeneration}\0${source.relayPtyId ?? ''}\0${source.clientGeneration}\0${source.ownerGeneration}\0${source.deliveryToken}` } } diff --git a/src/relay/dispatcher-client-writer.test.ts b/src/relay/dispatcher-client-writer.test.ts index 9218844bc1e..13e49b59e8e 100644 --- a/src/relay/dispatcher-client-writer.test.ts +++ b/src/relay/dispatcher-client-writer.test.ts @@ -300,24 +300,35 @@ describe('DispatcherClientWriter', () => { expect(sink.accepted[1].data).toBe('cancel') }) - it('admits one bounded producer frame larger than an empty native sink window', () => { + it('enforces the encoded HWM-minus-reserve limit for splittable producers', () => { const sink = new FakeSink() sink.highWaterMark = 4096 sink.saturateNext = true const writer = createWriter(sink, vi.fn(), 8192) + + expect(writer.producerFrameCapacity).toBe(3072) + expect(writer.enqueue('bulk', () => Buffer.alloc(3073), 3073)).toBe(false) + expect(writer.enqueue('bulk', () => Buffer.alloc(3072), 3072)).toBe(true) + }) + + it('admits only one indivisible fixed filesystem frame before drain', () => { + const sink = new FakeSink() + sink.highWaterMark = 4096 + sink.saturateNext = true + const writer = createWriter(sink, vi.fn(), 16_384) const oversized = 'x'.repeat(5000) expect(writer.producerFrameCapacity).toBe(3072) - const first = enqueue(writer, 'bulk', oversized) - enqueue(writer, 'ordinary', 'after-drain') - expect(writer.enqueue('bulk', () => Buffer.alloc(8193), 8193)).toBe(false) + expect(writer.enqueue('fixed-bulk', () => Buffer.from(oversized), 5000)).toBe(true) + expect(writer.enqueue('fixed-bulk', () => Buffer.from(oversized), 5000)).toBe(false) + expect(writer.enqueue('control', () => Buffer.from('cancel'), 6)).toBe(true) expect(sink.accepted.map((write) => write.data)).toEqual([oversized]) + sink.writableLength = 0 sink.accepted[0].settle({ ok: true }) - expect(first).toHaveBeenCalledWith({ ok: true }) - expect(sink.accepted).toHaveLength(1) + expect(writer.enqueue('fixed-bulk', () => Buffer.from(oversized), 5000)).toBe(false) sink.drain() - expect(sink.accepted.map((write) => write.data)).toEqual([oversized, 'after-drain']) + expect(sink.accepted.map((write) => write.data)).toEqual([oversized, 'cancel']) }) it('gives independent writers progress when one client stays saturated', () => { diff --git a/src/relay/dispatcher-client-writer.ts b/src/relay/dispatcher-client-writer.ts index d5c4898bb89..1966ce31dfc 100644 --- a/src/relay/dispatcher-client-writer.ts +++ b/src/relay/dispatcher-client-writer.ts @@ -59,8 +59,8 @@ export class DispatcherClientWriter { return this.sink.producerFrameCapacity } - get producerQueueCapacity(): number { - return this.admission.producerQueueCapacity + get fixedFrameCapacity(): number { + return this.sink.frameCapacity('fixed-bulk', this.saturated) } canEnqueueProducer(bytes: number): boolean { @@ -96,7 +96,7 @@ export class DispatcherClientWriter { onSettled: onceDispatcherWriterSettlement(onSettled), settled: false } - const admission = this.admission.admit(entry, this.producerFrameCapacity) + const admission = this.admission.admit(entry, this.sink.frameCapacity(lane, this.saturated)) if (!admission.accepted) { if (admission.error) { entry.onSettled({ ok: false, error: admission.error }) diff --git a/src/relay/dispatcher-writer-admission.ts b/src/relay/dispatcher-writer-admission.ts index 01634cbf91b..a49627d4dcd 100644 --- a/src/relay/dispatcher-writer-admission.ts +++ b/src/relay/dispatcher-writer-admission.ts @@ -4,6 +4,7 @@ export type DispatcherWriterLane = | 'legacy-response' | 'interactive' | 'ordinary' + | 'fixed-bulk' | 'bulk' export type DispatcherWriterSettlement = { ok: true } | { ok: false; error: Error } @@ -53,6 +54,7 @@ export class DispatcherWriterAdmission { 'legacy-response': [], interactive: [], ordinary: [], + 'fixed-bulk': [], bulk: [] } private controlBytes = 0 @@ -66,18 +68,13 @@ export class DispatcherWriterAdmission { return this.producerBytes } - get producerQueueCapacity(): number { - return this.producerQueueMaxBytes - } - get queuedEntries(): number { return Object.values(this.queues).reduce((total, queue) => total + queue.length, 0) } canAdmitProducer(bytes: number, producerFrameCapacity: number): boolean { return ( - bytes <= Math.max(producerFrameCapacity, this.producerQueueMaxBytes) && - this.producerBytes + bytes <= this.producerQueueMaxBytes + bytes <= producerFrameCapacity && this.producerBytes + bytes <= this.producerQueueMaxBytes ) } @@ -96,6 +93,9 @@ export class DispatcherWriterAdmission { this.queues[entry.lane].push(entry) return { accepted: true } } + if (entry.lane === 'fixed-bulk' && this.producerBytes > 0) { + return { accepted: false } + } if (!this.canAdmitProducer(entry.estimatedBytes, producerFrameCapacity)) { return { accepted: false } } diff --git a/src/relay/dispatcher-writer-lane-scheduler.ts b/src/relay/dispatcher-writer-lane-scheduler.ts index 966875dd7e9..dba1b254821 100644 --- a/src/relay/dispatcher-writer-lane-scheduler.ts +++ b/src/relay/dispatcher-writer-lane-scheduler.ts @@ -28,13 +28,13 @@ export class DispatcherWriterLaneScheduler { if (legacyResponse && canWriteControl(legacyResponse)) { return admission.shift('legacy-response') } - const bulk = admission.peek('bulk') + const bulk = admission.peek('fixed-bulk') ?? admission.peek('bulk') if ( bulk && this.producerWritesSinceBulk >= PRODUCER_WRITES_BEFORE_BULK && canWriteProducer(bulk) ) { - return admission.shift('bulk') + return admission.shift(bulk.lane) } const ordinary = admission.peek('ordinary') if ( @@ -44,7 +44,7 @@ export class DispatcherWriterLaneScheduler { ) { return admission.shift('ordinary') } - for (const lane of ['interactive', 'ordinary', 'bulk'] as const) { + for (const lane of ['interactive', 'ordinary', 'fixed-bulk', 'bulk'] as const) { const candidate = admission.peek(lane) if (candidate && canWriteProducer(candidate)) { return admission.shift(lane) @@ -54,7 +54,7 @@ export class DispatcherWriterLaneScheduler { } recordWrite(lane: DispatcherWriterLane): void { - if (lane === 'bulk') { + if (lane === 'bulk' || lane === 'fixed-bulk') { this.producerWritesSinceBulk = 0 } else if (lane === 'interactive' || lane === 'ordinary') { this.producerWritesSinceBulk = Math.min( diff --git a/src/relay/dispatcher-writer-sink.ts b/src/relay/dispatcher-writer-sink.ts index 7c06813155d..ef6c1349134 100644 --- a/src/relay/dispatcher-writer-sink.ts +++ b/src/relay/dispatcher-writer-sink.ts @@ -1,5 +1,6 @@ import { relayWriterControlReserve, + type DispatcherWriterLane, type DispatcherWriterSettlement } from './dispatcher-writer-admission' @@ -47,6 +48,13 @@ export class DispatcherWriterSink { return Math.max(0, this.highWaterMark - relayWriterControlReserve(this.highWaterMark)) } + frameCapacity(lane: DispatcherWriterLane, producerBlocked = false): number { + if (lane === 'fixed-bulk') { + return !producerBlocked && this.writableLength === 0 ? Number.MAX_SAFE_INTEGER : 0 + } + return this.producerFrameCapacity + } + registerDrain(callback: () => void): { registered: boolean; remove: () => void } { if (!this.options.waitWriteDrain) { return { registered: false, remove: () => {} } diff --git a/src/relay/dispatcher.ts b/src/relay/dispatcher.ts index f723899a63f..ae85d997b52 100644 --- a/src/relay/dispatcher.ts +++ b/src/relay/dispatcher.ts @@ -863,11 +863,14 @@ export class RelayDispatcher { private publishToClient( client: RelayClient, msg: JsonRpcNotification, - lane: 'interactive' | 'ordinary' | 'bulk', + lane: 'interactive' | 'ordinary' | 'fixed-bulk' | 'bulk', onSettled: (result: SinkWriteSettlement) => void = () => {} ): boolean { const bytes = this.estimateFrameBytes(msg) - if (!client.writer.canEnqueueProducer(bytes)) { + const fixedBlocked = + lane === 'fixed-bulk' && + (client.writer.retainedProducerBytes > 0 || bytes > client.writer.fixedFrameCapacity) + if (fixedBlocked || (lane !== 'fixed-bulk' && !client.writer.canEnqueueProducer(bytes))) { return false } const leases = this.publicationLedger.tryReserve([{ clientKey: this.clientKey(client), bytes }]) @@ -879,9 +882,13 @@ export class RelayDispatcher { private publishBulkWhenAvailable(client: RelayClient, msg: JsonRpcNotification): Promise { const bytes = this.estimateFrameBytes(msg) - if (bytes > client.writer.producerQueueCapacity) { + const lane = msg.method === 'fs.streamChunk' ? 'fixed-bulk' : 'bulk' + if (bytes > DEFAULT_PRODUCER_QUEUE_MAX_BYTES) { return Promise.reject(new Error('Relay bulk frame exceeds sink producer capacity')) } + if (lane === 'bulk' && bytes > client.writer.producerFrameCapacity) { + return Promise.reject(new Error('Relay bulk frame exceeds sink frame capacity')) + } return new Promise((resolve, reject) => { let removeCapacityListener: (() => void) | null = null const finish = (): void => { @@ -895,7 +902,7 @@ export class RelayDispatcher { return } if ( - this.publishToClient(client, msg, 'bulk', (result) => { + this.publishToClient(client, msg, lane, (result) => { finish() if (result.ok || this.disposed || client.closed) { resolve() @@ -917,14 +924,14 @@ export class RelayDispatcher { private enqueueLeasedFrame( client: RelayClient, msg: JsonRpcNotification, - lane: 'interactive' | 'ordinary' | 'bulk', + lane: 'interactive' | 'ordinary' | 'fixed-bulk' | 'bulk', lease: LegacyPublicationLease, onSettled: (result: SinkWriteSettlement) => void = () => {} ): boolean { const accepted = this.enqueueFrame(client, msg, lane, (result) => { lease.release() - this.notifyLegacyCapacityIfLow() onSettled(result) + this.notifyLegacyCapacityIfLow() }) if (!accepted) { lease.release() diff --git a/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts b/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts index 8cca2787891..98d95fd7f8a 100644 --- a/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts +++ b/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts @@ -28,6 +28,7 @@ import { import { readFileViaStream } from '../main/ssh/ssh-filesystem-stream-reader' import { RelayDispatcher } from './dispatcher' +import type { SinkWriteSettlement } from './dispatcher' import { RelayContext } from './context' import { FsHandler } from './fs-handler' import { STREAM_CHUNK_SIZE } from './protocol' @@ -95,7 +96,10 @@ function createHarness(opts: { congested: boolean }): Harness { onClose: () => {} } - const outQueue: Buffer[] = [] + const outQueue: { + data: Buffer + settle: (result: SinkWriteSettlement) => void + }[] = [] let queuedBytes = 0 const drainWaiters = new Set<() => void>() const fireDrainIfIdle = (): void => { @@ -109,8 +113,8 @@ function createHarness(opts: { congested: boolean }): Harness { } const dispatcher = new RelayDispatcher( - (data: Buffer) => { - outQueue.push(data) + (data: Buffer, settle) => { + outQueue.push({ data, settle }) queuedBytes += data.length if (!opts.congested) { return true @@ -118,6 +122,9 @@ function createHarness(opts: { congested: boolean }): Harness { return queuedBytes < SINK_HIGH_WATER_MARK }, { + supportsWriteCallback: true, + writableLength: () => queuedBytes, + writableHighWaterMark: () => SINK_HIGH_WATER_MARK, waitWriteDrain: (cb: () => void) => { drainWaiters.add(cb) fireDrainIfIdle() @@ -128,11 +135,12 @@ function createHarness(opts: { congested: boolean }): Harness { const deliverAll = (): void => { while (outQueue.length > 0) { - const buf = outQueue.shift()! - queuedBytes -= buf.length + const { data, settle } = outQueue.shift()! + queuedBytes -= data.length for (const cb of clientDataCallbacks) { - cb(buf) + cb(data) } + settle({ ok: true }) } fireDrainIfIdle() } @@ -207,7 +215,7 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { // The echo must not sit behind an unbounded chunk backlog: at most one // in-flight bulk frame (the write that saturated the sink) plus slack. - expect(queuedBytesAheadOfEcho).toBeLessThan(2 * FRAMED_CHUNK_BYTES) + expect(queuedBytesAheadOfEcho).toBeLessThan(FRAMED_CHUNK_BYTES) // Un-congest: the stream must still complete with intact content. harness.startAutoDeliver() diff --git a/src/relay/git-response-pty-echo-backpressure.integration.test.ts b/src/relay/git-response-pty-echo-backpressure.integration.test.ts index 09e6bf38192..4f3eee4a87a 100644 --- a/src/relay/git-response-pty-echo-backpressure.integration.test.ts +++ b/src/relay/git-response-pty-echo-backpressure.integration.test.ts @@ -28,6 +28,7 @@ import { import { requestGitStreamable } from '../main/ssh/ssh-git-response-stream-reader' import { RelayDispatcher } from './dispatcher' +import type { SinkWriteSettlement } from './dispatcher' import { RelayContext } from './context' import { GitHandler } from './git-handler' import { GIT_RESPONSE_CHUNK_SIZE } from './protocol' @@ -91,7 +92,10 @@ function createHarness(opts: { congested: boolean }): Harness { onClose: () => {} } - const outQueue: Buffer[] = [] + const outQueue: { + data: Buffer + settle: (result: SinkWriteSettlement) => void + }[] = [] let queuedBytes = 0 const drainWaiters = new Set<() => void>() const fireDrainIfIdle = (): void => { @@ -105,8 +109,8 @@ function createHarness(opts: { congested: boolean }): Harness { } const dispatcher = new RelayDispatcher( - (data: Buffer) => { - outQueue.push(data) + (data: Buffer, settle) => { + outQueue.push({ data, settle }) queuedBytes += data.length if (!opts.congested) { return true @@ -114,6 +118,9 @@ function createHarness(opts: { congested: boolean }): Harness { return queuedBytes < SINK_HIGH_WATER_MARK }, { + supportsWriteCallback: true, + writableLength: () => queuedBytes, + writableHighWaterMark: () => SINK_HIGH_WATER_MARK, waitWriteDrain: (cb: () => void) => { drainWaiters.add(cb) fireDrainIfIdle() @@ -124,11 +131,12 @@ function createHarness(opts: { congested: boolean }): Harness { const deliverAll = (): void => { while (outQueue.length > 0) { - const buf = outQueue.shift()! - queuedBytes -= buf.length + const { data, settle } = outQueue.shift()! + queuedBytes -= data.length for (const cb of clientDataCallbacks) { - cb(buf) + cb(data) } + settle({ ok: true }) } fireDrainIfIdle() } diff --git a/src/relay/protocol-backpressure.test.ts b/src/relay/protocol-backpressure.test.ts index a6569ec8ce9..49a31a01c0a 100644 --- a/src/relay/protocol-backpressure.test.ts +++ b/src/relay/protocol-backpressure.test.ts @@ -41,6 +41,27 @@ function frame(id: number, payload = `${id}`): Buffer { } describe('relay FrameDecoder bounded turns', () => { + it('retains at most one maximum frame plus one MiB of partial input', () => { + expect(FRAME_DECODER_MAX_RETAINED_BYTES).toBe(MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024) + const maximumFrame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(MAX_MESSAGE_SIZE)) + const acceptedError = vi.fn() + const accepted = new FrameDecoder(vi.fn(), acceptedError) + + accepted.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024)])) + + expect(acceptedError).not.toHaveBeenCalled() + expect(accepted.drain()).toHaveLength(1024 * 1024) + + const excessError = vi.fn() + const excess = new FrameDecoder(vi.fn(), excessError) + excess.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024 + 1)])) + + expect(excessError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('retained-input') }) + ) + expect(excess.drain()).toHaveLength(0) + }) + it('fails closed when one delivered chunk exceeds retained-input capacity', () => { const onError = vi.fn() const decoder = new FrameDecoder(vi.fn(), onError) diff --git a/src/relay/pty-handler-source-publication.test.ts b/src/relay/pty-handler-source-publication.test.ts new file mode 100644 index 00000000000..eb620cf59ea --- /dev/null +++ b/src/relay/pty-handler-source-publication.test.ts @@ -0,0 +1,257 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PTY_STARTUP_INGRESS_VERSION } from '../shared/pty-startup-ingress' +import { + RelayDispatcher, + type RelayClientSessionIdentity, + type SinkWriteSettlement +} from './dispatcher' +import { encodeJsonRpcFrame, MessageType } from './protocol' +import { PtyHandler } from './pty-handler' +import { RelayPtySourcePublication } from './relay-pty-source-publication' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +const { mockPtySpawn } = vi.hoisted(() => ({ mockPtySpawn: vi.fn() })) + +vi.mock('node-pty', () => ({ spawn: mockPtySpawn })) + +const endpointIdentity: RelayClientSessionIdentity = { + principal: 'endpoint-principal', + authenticated: true, + allowSessionOwner: true, + authenticationKind: 'endpoint-credential' +} + +type Notification = { method: string; params: Record } + +function requestFrame(id: number, method: string, params: Record): Buffer { + return encodeJsonRpcFrame({ jsonrpc: '2.0', id, method, params }, id, 0) +} + +function notification(buffer: Buffer): Notification | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + const message = JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) + return typeof message.method === 'string' && message.id === undefined ? message : null +} + +function responseResult(buffer: Buffer, id: number): Record | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + const message = JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) + return message.id === id ? (message.result ?? null) : null +} + +describe('PtyHandler negotiated source publication', () => { + let dispatcher: RelayDispatcher + let handler: PtyHandler + let publication: RelayPtySourcePublication + let dataCallback: ((data: string) => void) | undefined + let originalPlatform: PropertyDescriptor | undefined + let writes: Buffer[] + let heldResponseId: number | null + let heldResponseSettlements: ((result: SinkWriteSettlement) => void)[] + let adapter: SshPtyConsumerSessionAdapter + + beforeEach(async () => { + vi.useFakeTimers() + originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + writes = [] + heldResponseId = null + heldResponseSettlements = [] + dataCallback = undefined + mockPtySpawn.mockReset() + mockPtySpawn.mockReturnValue({ + pid: process.pid, + onData: vi.fn((callback: (data: string) => void) => { + dataCallback = callback + }), + onExit: vi.fn(), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(), + clear: vi.fn(), + pause: vi.fn(), + resume: vi.fn() + }) + dispatcher = new RelayDispatcher( + (data, settle) => { + writes.push(Buffer.from(data)) + if (heldResponseId !== null && responseResult(data, heldResponseId)) { + heldResponseSettlements.push(settle) + return true + } + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + handler = new PtyHandler(dispatcher) + adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, (id) => + handler.handleSourcePublicationCapacity(id) + ) + handler.setSourcePublication(publication) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 1024 } } + }) + ) + await vi.advanceTimersByTimeAsync(0) + }) + + afterEach(async () => { + await handler.dispose({ waitForPhysicalExit: false }).catch(() => {}) + dispatcher.dispose() + if (originalPlatform) { + Object.defineProperty(process, 'platform', originalPlatform) + } + vi.useRealTimers() + }) + + async function spawn(params: Record): Promise { + dispatcher.feed(requestFrame(2, 'pty.spawn', params)) + await vi.advanceTimersByTimeAsync(0) + expect(dataCallback).toBeTypeOf('function') + } + + function sourceDataFrames(): Notification[] { + return writes + .map(notification) + .filter((frame): frame is Notification => frame?.method === 'pty.data') + } + + it('settles a consumed POSIX startup query before publishing its prompt', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) + await spawn({ + startupIngressVersion: PTY_STARTUP_INGRESS_VERSION, + startupIngress: { + colors: { foreground: '#2e3434', background: '#ffffff' }, + deadlineMs: 5_000 + } + }) + const query = '\x1b]10;?\x07' + + dataCallback!(query) + dataCallback!('prompt') + await vi.advanceTimersByTimeAsync(9) + + expect(sourceDataFrames().map((frame) => frame.params)).toEqual([ + expect.objectContaining({ + data: '', + rawLength: query.length, + transformed: true, + sourceLengthSu: query.length, + sourceEndSu: query.length + }), + expect.objectContaining({ + data: 'prompt', + rawLength: 6, + sourceLengthSu: 6, + sourceEndSu: query.length + 6 + }) + ]) + expect(publication.getDebugSnapshot()).toMatchObject({ sendCommitted: 2 }) + }) + + it('settles a suppressed ConPTY query before publishing its prompt', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + await spawn({ shellOverride: 'powershell.exe' }) + const query = '\x1b]10;?\x07' + + dataCallback!(query) + dataCallback!('PS> ') + await vi.advanceTimersByTimeAsync(9) + + expect(sourceDataFrames().map((frame) => frame.params)).toEqual([ + expect.objectContaining({ + data: '', + rawLength: query.length, + transformed: true, + sourceLengthSu: query.length, + sourceEndSu: query.length + }), + expect.objectContaining({ + data: 'PS> ', + rawLength: 4, + sourceLengthSu: 4, + sourceEndSu: query.length + 4 + }) + ]) + expect(publication.getDebugSnapshot()).toMatchObject({ sendCommitted: 2 }) + }) + + it('activates a fresh source consumer when an operation response is lost and retried', async () => { + const operationId = 'a'.repeat(43) + const grant = writes.map((buffer) => responseResult(buffer, 1)).find(Boolean)! + heldResponseId = 2 + dispatcher.feed( + requestFrame(2, 'pty.spawn', { + agentSessionCreateOperationId: operationId + }) + ) + await vi.advanceTimersByTimeAsync(0) + expect(heldResponseSettlements).toHaveLength(1) + + dispatcher.invalidateClient() + expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 0 }) + + const replacementWrites: Buffer[] = [] + const replacementClientId = dispatcher.attachClient( + (data, settle) => { + replacementWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + replacementClientId, + requestFrame(3, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: grant.ownerGeneration, + ownerLease: grant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 1024 } } + }) + ) + await vi.advanceTimersByTimeAsync(0) + dispatcher.feedClient( + replacementClientId, + requestFrame(4, 'pty.spawn', { + agentSessionCreateOperationId: operationId + }) + ) + await vi.advanceTimersByTimeAsync(0) + + dataCallback!('prompt') + await vi.advanceTimersByTimeAsync(8) + + expect(mockPtySpawn).toHaveBeenCalledOnce() + expect( + responseResult(replacementWrites.find((buffer) => responseResult(buffer, 4))!, 4) + ).toEqual({ id: 'pty-1', incarnationId: expect.any(String) }) + expect( + replacementWrites.map(notification).find((frame) => frame?.method === 'pty.data')?.params + ).toMatchObject({ + id: 'pty-1', + data: 'prompt', + sourceLengthSu: 6, + sourceEndSu: 6 + }) + expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1 }) + }) +}) diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 32d34210032..981fd64cd67 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -876,6 +876,8 @@ export class PtyHandler { const desiredChars = pending.transformed ? pending.data.length : Math.min(pending.data.length, PTY_OUTPUT_FLUSH_CHUNK_CHARS) + const sourceOnlyEmission = + pending.transformed === true && pending.data.length === 0 && (pending.rawLength ?? 0) > 0 const paramsWithoutData = { id, ...(pending.seq === undefined ? {} : { seq: pending.seq }), @@ -894,7 +896,10 @@ export class PtyHandler { ) { chunkChars-- } - if (chunkChars <= 0 || (pending.transformed && chunkChars !== pending.data.length)) { + if ( + (!sourceOnlyEmission && chunkChars <= 0) || + (pending.transformed && chunkChars !== pending.data.length) + ) { this.pendingOutputByPty.set(id, queue) this.pausePtyOutput(id) return false @@ -1137,7 +1142,9 @@ export class PtyHandler { } const existing = this.agentSessionCreateOperations.get(operationId) if (existing) { - return await existing + const result = await existing + this.sourcePublication?.activate(result.id, result.incarnationId, context) + return result } if (this.agentSessionCreateOperations.size >= AGENT_SESSION_CREATE_OPERATION_LIMIT) { throw new Error('agent_session_operation_capacity') diff --git a/src/relay/pty-source-credit-ledger.test.ts b/src/relay/pty-source-credit-ledger.test.ts index 384722210b2..f8020a3914c 100644 --- a/src/relay/pty-source-credit-ledger.test.ts +++ b/src/relay/pty-source-credit-ledger.test.ts @@ -355,6 +355,66 @@ describe('RelayPtySourceCreditLedger', () => { expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 0, dataBytes: 0, spans: 0 }) }) + it('charges UTF-16 storage and one record per retained source frame', () => { + const ledger = new RelayPtySourceCreditLedger({ + maxRetainedDataBytes: 391, + maxAggregateRetainedDataBytes: 391 + }) + const owner = identity() + ledger.open(owner, 8) + append(ledger, owner, '\u0000'.repeat(64), 'utf16-heavy') + + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 64, dataBytes: 256, spans: 1 }) + expect(() => append(ledger, owner, 'a'.repeat(8), 'fragment')).toThrow('retained encoded-data') + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 64, dataBytes: 256, spans: 1 }) + }) + + it('enforces per-PTY and aggregate source, charged-byte, and frame caps across fragments', () => { + const limits = { + maxRetainedSourceSu: 6, + maxAggregateRetainedSourceSu: 9, + maxRetainedDataBytes: 500, + maxAggregateRetainedDataBytes: 700, + maxRetainedSpans: 2, + maxAggregateRetainedSpans: 3 + } + const first = identity('token-a') + const second = identity('token-b', { + id: 'pty-2', + ptyIncarnation: 'incarnation-2' + }) + const ledger = new RelayPtySourceCreditLedger(limits) + ledger.open(first, 8) + ledger.open(second, 8) + append(ledger, first, 'aa', 'first-a') + append(ledger, first, 'bb', 'first-b') + append(ledger, second, 'cc', 'second-a') + + expect(ledger.retentionSnapshot()).toEqual({ + sourceSu: 6, + dataBytes: 396, + spans: 3 + }) + expect(() => append(ledger, first, 'c', 'first-frame-cap')).toThrow('retained-span') + expect(() => append(ledger, second, 'dddd', 'aggregate-source-cap')).toThrow('Aggregate') + + const byteLimited = new RelayPtySourceCreditLedger({ + ...limits, + maxRetainedSourceSu: 200, + maxAggregateRetainedSourceSu: 300, + maxAggregateRetainedSpans: 10, + maxRetainedSpans: 10, + maxAggregateRetainedDataBytes: 650 + }) + byteLimited.open(first, 8) + byteLimited.open(second, 8) + append(byteLimited, first, '\u0000'.repeat(64), 'first-utf16') + append(byteLimited, second, '\u0000'.repeat(64), 'second-utf16') + expect(() => append(byteLimited, second, '\u0000'.repeat(8), 'aggregate-byte-cap')).toThrow( + 'Aggregate' + ) + }) + it('bounds retained frame metadata independently of source units', () => { const ledger = new RelayPtySourceCreditLedger({ maxRetainedSpans: 2, @@ -366,7 +426,7 @@ describe('RelayPtySourceCreditLedger', () => { append(ledger, owner, 'b', 'span-b') expect(() => append(ledger, owner, 'c', 'span-c')).toThrow('retained-span') - expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 2, dataBytes: 2, spans: 2 }) + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 2, dataBytes: 260, spans: 2 }) }) it('bounds closed token tombstones', () => { diff --git a/src/relay/pty-source-credit-ledger.ts b/src/relay/pty-source-credit-ledger.ts index 0360392cfb9..f605f390bad 100644 --- a/src/relay/pty-source-credit-ledger.ts +++ b/src/relay/pty-source-credit-ledger.ts @@ -38,6 +38,7 @@ import { type PtySourceAppendInput, type PtySourceSendReservation } from './pty-source-credit-record' +import { chargedPtyRetainedStringBytes } from '../shared/pty-retained-string-memory' export type { PtySourceSendReservation } from './pty-source-credit-record' export type { PtySourceCreditLedgerOptions } from './pty-source-credit-limits' @@ -78,8 +79,9 @@ export class RelayPtySourceCreditLedger { append(identity: PtySourceDeliveryIdentity, input: PtySourceAppendInput): PtySourceSpan { const record = this.requireActive(identity) const sourceEndSu = record.receivedEndSu + input.transform.rawLengthSu - const dataBytes = Buffer.byteLength(input.data, 'utf8') - if (dataBytes > MAX_SOURCE_SPAN_DATA_BYTES) { + const encodedDataBytes = Buffer.byteLength(input.data, 'utf8') + const retainedDataBytes = chargedPtyRetainedStringBytes(input.data) + if (encodedDataBytes > MAX_SOURCE_SPAN_DATA_BYTES) { throw new Error('PTY source span encoded-data budget exceeded') } if (sourceEndSu - record.creditedEndSu > this.limits.maxRetainedSourceSu) { @@ -91,10 +93,10 @@ export class RelayPtySourceCreditLedger { ) { throw new Error('Aggregate PTY source retained-range budget exceeded') } - if (record.retainedDataBytes + dataBytes > this.limits.maxRetainedDataBytes) { + if (record.retainedDataBytes + retainedDataBytes > this.limits.maxRetainedDataBytes) { throw new Error('PTY source retained encoded-data budget exceeded') } - if (this.retainedDataBytes() + dataBytes > this.limits.maxAggregateRetainedDataBytes) { + if (this.retainedDataBytes() + retainedDataBytes > this.limits.maxAggregateRetainedDataBytes) { throw new Error('Aggregate PTY source retained encoded-data budget exceeded') } if (record.spans.length + 1 > this.limits.maxRetainedSpans) { @@ -105,7 +107,7 @@ export class RelayPtySourceCreditLedger { } const span = createAppendedSourceSpan(record, input) record.spans.push(span) - record.retainedDataBytes += dataBytes + record.retainedDataBytes += retainedDataBytes record.receivedEndSu = sourceEndSu return span } diff --git a/src/relay/pty-source-credit-record.ts b/src/relay/pty-source-credit-record.ts index 985369bbc4c..a2b1ec94060 100644 --- a/src/relay/pty-source-credit-record.ts +++ b/src/relay/pty-source-credit-record.ts @@ -14,6 +14,7 @@ import { assertPtySourceIdentity, assertPtySourceSpan } from '../shared/pty-source-credit-validation' +import { chargedPtyRetainedStringBytes } from '../shared/pty-retained-string-memory' export const DEFAULT_RETAINED_SOURCE_SU = 512 * 1024 export const DEFAULT_AGGREGATE_RETAINED_SOURCE_SU = 48 * 1024 * 1024 @@ -172,7 +173,7 @@ export function snapshotDeliveryRecord(record: DeliveryRecord): PtySourceDeliver export function reclaimCreditedSpans(record: DeliveryRecord): void { while (record.spans[0]?.sourceEndSu <= record.creditedEndSu) { - record.retainedDataBytes -= Buffer.byteLength(record.spans.shift()!.data, 'utf8') + record.retainedDataBytes -= chargedPtyRetainedStringBytes(record.spans.shift()!.data) } } @@ -272,7 +273,7 @@ export function createReplacementDeliveryRecord( .map((span) => sliceAtSourceStart(span, Math.max(span.sourceStartSu, acceptedSourceEndSu))) .map((span) => Object.freeze({ ...span, ...replacement.identity })) replacement.retainedDataBytes = replacement.spans.reduce( - (bytes, span) => bytes + Buffer.byteLength(span.data, 'utf8'), + (bytes, span) => bytes + chargedPtyRetainedStringBytes(span.data), 0 ) return replacement diff --git a/src/relay/relay-frame-decoder.ts b/src/relay/relay-frame-decoder.ts index f6673d8b6ac..11cfb08311f 100644 --- a/src/relay/relay-frame-decoder.ts +++ b/src/relay/relay-frame-decoder.ts @@ -6,7 +6,7 @@ export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 export const FRAME_DECODER_MAX_FRAMES_PER_TURN = 64 export const FRAME_DECODER_MAX_BYTES_PER_TURN = MAX_MESSAGE_SIZE + HEADER_LENGTH export const FRAME_DECODER_MAX_TURN_MS = 4, - FRAME_DECODER_MAX_RETAINED_BYTES = 2 * (MAX_MESSAGE_SIZE + HEADER_LENGTH) + FRAME_DECODER_MAX_RETAINED_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024 export class FrameDecoder { private chunks: Buffer[] = [] diff --git a/src/relay/relay-pty-source-publication.test.ts b/src/relay/relay-pty-source-publication.test.ts index 65248d53111..dc9cce19738 100644 --- a/src/relay/relay-pty-source-publication.test.ts +++ b/src/relay/relay-pty-source-publication.test.ts @@ -49,7 +49,11 @@ describe('RelayPtySourcePublication', () => { dispatcher = null }) - async function createHarness(windowSu = 4, settleSourceImmediately = true) { + async function createHarness( + windowSu = 4, + settleSourceImmediately = true, + highWaterMark?: number + ) { const writes: Buffer[] = [] const sourceSettlements: ((result: SinkWriteSettlement) => void)[] = [] dispatcher = new RelayDispatcher( @@ -66,7 +70,15 @@ describe('RelayPtySourcePublication', () => { } return true }, - { supportsWriteCallback: true }, + { + supportsWriteCallback: true, + ...(highWaterMark + ? { + writableLength: () => 0, + writableHighWaterMark: () => highWaterMark + } + : {}) + }, endpointIdentity ) let publication: RelayPtySourcePublication @@ -137,6 +149,27 @@ describe('RelayPtySourcePublication', () => { await flushRequests() }) + it('slices source frames to the encoded HWM-minus-reserve capacity', async () => { + const highWaterMark = 4096 + const harness = await createHarness(10_000, true, highWaterMark) + const payload = '\u0000'.repeat(4000) + + expect(harness.publication.publish('pty-1', { data: payload }, false)).toBe(true) + for (let turn = 0; turn < 4; turn++) { + await flushRequests() + } + + const frames = harness.writes.filter((buffer) => notification(buffer)?.method === 'pty.data') + expect(frames.length).toBeGreaterThan(1) + expect(Math.max(...frames.map((buffer) => buffer.length))).toBeLessThanOrEqual(3072) + expect( + frames + .map(notification) + .map((frame) => frame!.params.data) + .join('') + ).toBe(payload) + }) + it('keeps mixed legacy and V1 clients on distinct frame authority', async () => { const harness = await createHarness(8) const legacyWrites: Buffer[] = [] diff --git a/src/relay/relay-pty-source-send-scheduler.ts b/src/relay/relay-pty-source-send-scheduler.ts index e8c0676a062..6622e1e7a6e 100644 --- a/src/relay/relay-pty-source-send-scheduler.ts +++ b/src/relay/relay-pty-source-send-scheduler.ts @@ -35,6 +35,8 @@ export type RelayPtySourcePublicationCounters = { exitRolledBack: number } +const PTY_SOURCE_FRAME_MAX_SU = 16 * 1024 + export function onceSinkSettlement( callback: (result: SinkWriteSettlement) => void ): (result: SinkWriteSettlement) => void { @@ -153,7 +155,27 @@ export class RelayPtySourceSendScheduler { }) return } - const reservation = this.session.reserveSourceSend(record.identity) + const snapshot = this.session.sourceDeliverySnapshot(record.identity) + const encodedDataBudget = this.dispatcher.producerDataBudget( + 'pty.data', + { + id: record.identity.id, + rawLength: PTY_SOURCE_FRAME_MAX_SU, + transformed: false, + deliveryToken: record.identity.deliveryToken, + clientGeneration: record.identity.clientGeneration, + ownerGeneration: record.identity.ownerGeneration, + ptyIncarnation: record.identity.ptyIncarnation, + sourceEndSu: snapshot.receivedEndSu, + sourceLengthSu: PTY_SOURCE_FRAME_MAX_SU + }, + record.clientId + ) + const maxSourceSu = Math.min( + PTY_SOURCE_FRAME_MAX_SU, + Math.max(1, Math.floor(Math.max(0, encodedDataBudget - 32) / 6)) + ) + const reservation = this.session.reserveSourceSend(record.identity, maxSourceSu) if (!reservation) { return } diff --git a/src/shared/pty-consumer-session-contract.ts b/src/shared/pty-consumer-session-contract.ts index fb068ce0e51..661138896cd 100644 --- a/src/shared/pty-consumer-session-contract.ts +++ b/src/shared/pty-consumer-session-contract.ts @@ -1,5 +1,6 @@ export const PTY_CONSUMER_SESSION_PROTOCOL_VERSION = 1 export const PTY_CONSUMER_OWNER_GRACE_MS = 30_000 +export const PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR = -32041 export type PtyConsumerRole = 'session-owner' | 'subscriber' diff --git a/src/shared/pty-consumer-session.test.ts b/src/shared/pty-consumer-session.test.ts index 3ec266f9902..07c9c12f857 100644 --- a/src/shared/pty-consumer-session.test.ts +++ b/src/shared/pty-consumer-session.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { + PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, PtyConsumerSession, type PtyConsumerAuthentication, type PtyConsumerSessionHello @@ -37,6 +38,22 @@ function createSession(options: { now?: () => number } = {}): PtyConsumerSession } describe('PtyConsumerSession', () => { + it('types a resume against a fresh relay without weakening other owner refusals', () => { + const session = createSession() + + expect(() => + session.admit( + ownerHello({ resume: { ownerGeneration: 1, ownerLease: 'stale' } }), + auth('connection-1') + ) + ).toThrow( + expect.objectContaining({ + code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, + message: expect.stringContaining('stale') + }) + ) + }) + it('activates an authenticated owner only after its publication fence', () => { const session = createSession() const first = session.admit(ownerHello(), auth('connection-1')) diff --git a/src/shared/pty-consumer-session.ts b/src/shared/pty-consumer-session.ts index c6d83d0fb77..a91e9b18fa1 100644 --- a/src/shared/pty-consumer-session.ts +++ b/src/shared/pty-consumer-session.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import { PTY_CONSUMER_OWNER_GRACE_MS, PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, type PtyConsumerAuthentication, type PtyConsumerSessionAdmission, type PtyConsumerSessionGrant, @@ -244,7 +245,9 @@ export class PtyConsumerSession { const current = this.owner if (!current) { if (hello.resume) { - throw new Error('Owner recovery lease is stale') + throw Object.assign(new Error('Owner recovery lease is stale'), { + code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR + }) } return this.newOwner(hello, authentication, null) } diff --git a/src/shared/pty-retained-string-memory.ts b/src/shared/pty-retained-string-memory.ts new file mode 100644 index 00000000000..a6d1143dbb1 --- /dev/null +++ b/src/shared/pty-retained-string-memory.ts @@ -0,0 +1,5 @@ +export const PTY_RETAINED_RECORD_BYTES = 128 + +export function chargedPtyRetainedStringBytes(value: string): number { + return Math.max(Buffer.byteLength(value, 'utf8'), 2 * value.length) + PTY_RETAINED_RECORD_BYTES +}