diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 10829928880..abd1c210726 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -2,13 +2,7 @@ "schemaVersion": 1, "updatedAt": "2026-07-28", "policy": { - "maturityLevels": [ - "experimental", - "soak", - "blocking", - "accepted-gap", - "deprecated" - ], + "maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"], "blockingPromotion": { "minimumSoakRuns": 100, "minimumSoakDays": 14, @@ -29,40 +23,23 @@ "host removal confirmation", "shared mobile bottom drawers" ], - "platforms": [ - "ios", - "android", - "macos" - ], - "providers": [ - "provider-independent" - ], - "coveredPlatforms": [ - "ios", - "macos" - ], - "coveredProviders": [ - "provider-independent" - ], + "platforms": ["ios", "android", "macos"], + "providers": ["provider-independent"], + "coveredPlatforms": ["ios", "macos"], + "coveredProviders": ["provider-independent"], "coverageNotes": "A deterministic React lifecycle test proves callback churn cannot restart an in-flight drawer close, and an iOS 26.5 simulator run covers Edit Host plus confirmed host removal. Android native-modal behavior remains a live-test gap.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/8791" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/8791"], "invariant": "Once a bottom drawer begins closing, unrelated parent rerenders must not replace its completion callback or restart the native hide animation. The drawer must unmount once and deliver the latest after-close action exactly once.", "oracle": "Render one drawer, begin closing it, rerender with new parent callbacks before completion, and require every MountedBottomDrawer frame to retain one onHidden identity. Trigger that completion barrier repeatedly, then require the drawer's null render to commit before only the latest after-close callback runs exactly once.", "commands": [ "pnpm --dir mobile exec vitest run --root .. mobile/src/components/bottom-drawer-close-lifecycle.test.ts", "Manual iOS 26.5 simulator: long-press paired host; open Edit host; return; long-press host; Remove; confirm Remove; assert host disappears" ], - "testFiles": [ - "mobile/src/components/bottom-drawer-close-lifecycle.test.ts" - ], + "testFiles": ["mobile/src/components/bottom-drawer-close-lifecycle.test.ts"], "assertionRefs": [ { "file": "mobile/src/components/bottom-drawer-close-lifecycle.test.ts", - "assertions": [ - "keeps close stable and delivers the latest action once after unmount" - ] + "assertions": ["keeps close stable and delivers the latest action once after unmount"] } ], "evidenceRuns": [ @@ -126,30 +103,12 @@ "LAN direct endpoint", "Tailscale direct endpoint" ], - "platforms": [ - "ios", - "android", - "macos", - "linux", - "windows" - ], - "providers": [ - "lan", - "tailscale", - "cloud-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "lan", - "tailscale", - "cloud-relay" - ], + "platforms": ["ios", "android", "macos", "linux", "windows"], + "providers": ["lan", "tailscale", "cloud-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["lan", "tailscale", "cloud-relay"], "coverageNotes": "Deterministic TypeScript tests cover shared close-code policy, foreground retry timers, direct-winner cancellation, and concurrent LAN/Tailscale authentication. Physical iOS/Android radios, GFE, and production relay recovery remain live-test gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca-cloud/pull/96" - ], + "motivatingLinks": ["https://github.com/stablyai/orca-cloud/pull/96"], "invariant": "A foregrounded paired phone must recover from relay HOST_OFFLINE without a foreground or network-change signal, while direct recovery must select the first authenticated configured LAN or Tailscale endpoint without serial timeout delays. Backgrounding, direct success, or stop must cancel pending work, and losing probes must close without affecting the winner.", "oracle": "Inject deterministic relay close codes, random bytes, fake timers, and independently controlled direct clients. Require HOST_OFFLINE to replace any faster transport timer with one 5-15 second retry, require no retry before the selected delay, race all unique non-relay endpoints, select the first authenticated path, close every loser exactly once, and retain no retry after direct connectivity wins.", "commands": [ @@ -185,9 +144,7 @@ }, { "file": "src/shared/mobile-relay-close-codes.test.ts", - "assertions": [ - "HOST_OFFLINE maps to self-healing full-jitter recovery" - ] + "assertions": ["HOST_OFFLINE maps to self-healing full-jitter recovery"] } ], "evidenceRuns": [ @@ -250,24 +207,12 @@ "director assignment overload", "relay broker shutdown" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "cloud-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "cloud-relay" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["cloud-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["cloud-relay"], "coverageNotes": "Deterministic main-process tests cover duplicate drain notifications, full-jitter backoff, Retry-After during initial setup and drain recovery, successful recovery, and broker-close cleanup. Packaged desktop, mixed-version fleets, GFE, and production Cloud SQL remain live-test gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca-cloud/actions/runs/30223521062" - ], + "motivatingLinks": ["https://github.com/stablyai/orca-cloud/actions/runs/30223521062"], "invariant": "One relay host may have at most one assignment attempt or retry timer per recovery path. Sustained director failure must increase the retry window up to five minutes, a bounded Retry-After must be respected during initial setup and drain recovery, shutdown must cancel pending work, and recovery must activate the authoritative assigned origin.", "oracle": "Inject duplicate drain events, deterministic randomness, fake time, repeated assignment failures, a 30-second Retry-After during initial setup and drain recovery, broker close, and eventual director recovery. Count every assignment call, require 500 ms then 1,000 ms retry windows, reject duplicate fanout, require no pre-hint retry or post-close work, and prove the recovered cell becomes authoritative.", "commands": [ @@ -290,15 +235,11 @@ }, { "file": "src/main/runtime/relay/relay-http-client.test.ts", - "assertions": [ - "assignment overload preserves a bounded Retry-After hint" - ] + "assertions": ["assignment overload preserves a bounded Retry-After hint"] }, { "file": "src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts", - "assertions": [ - "initial relay setup does not retry before Retry-After expires" - ] + "assertions": ["initial relay setup does not retry before Retry-After expires"] } ], "evidenceRuns": [ @@ -356,26 +297,12 @@ "direct SSH host catalog authority", "direct SSH generation-scope rollover" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "ssh"], "coverageNotes": "Local deterministic evidence covers git-common classification, desktop watcher debounce counts, non-overlapping poller semantics, macOS native-watch fallback, preload cleanup, Source Control active-visible repo filtering, the direct SSH five-slot fair scheduler, timeout barrier, aggregate privacy schema, coordinator-to-renderer telemetry wiring, host-catalog provenance rejection, and process generation-scope rollover across sibling targets. A macOS Electron client completed a direct SSH disconnect/reconnect against an ephemeral Linux Docker target with exact host/authority hydration and remote proof-file verification. Linux/Windows desktop clients, multi-target live fanout, paired-client, and WSL runs remain gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/7086" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/7086"], "invariant": "Index-only Git activity below the common Git directory must not emit worktrees:changed, invalidate worktree caches, or trigger fetchWorktrees fanout; structural add/remove/HEAD/gitdir/locked/config.worktree changes must still refresh worktrees and nudge Source Control; external head moves (commit, amend, reset) must reach background worktree rows through spawn-free metadata reads, never through structural fanout. Direct SSH reconnect discovery must stay host- and authority-qualified, reject contradictory main-catalog provenance without returning rows, admit at most five locally unsettled provider calls, retain a retrying timeout barrier, and emit one identifier-free aggregate product event per target operation. A process generation-scope rollover revokes every direct SSH target and old-scope provider request, not only the target whose counter exhausted.", "oracle": "Classify exact git-common paths as structural, status-only, or ignored; count notifications from debounced watcher events; force the Linux/Windows poll path to emit allowlisted leaf events, detect linked HEAD rewrites independent of entry-directory mtime, and surface in-place index rewrites via the backstop re-stat; diff head identities from metadata-file reads and notify only real head moves; assert Source Control subscribes to both structural and status-only signals with active-repo and visibility filters. For direct SSH, reject catalog rows whose explicit and legacy host provenance contradict, roll one exhausted target into a fresh process generation scope while invalidating sibling target tokens, count locally unsettled attempts and round-robin admissions, keep lineage blocked through the first timeout retry, distinguish timeout/rejection/cancel/stale results, and reject telemetry properties carrying target, repo, host, path, label, user, request, lease, terminal, or raw-error data.", "commands": [ @@ -602,24 +529,10 @@ "persistent terminal reattach", "update install handoff" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "Deterministic unit coverage exercises activation gating, single-instance ownership, quit policy, local/remote CLI status, headless binding persistence, local daemon identity, SSH identity transfer, and the macOS serve update handoff from staged installer through atomic bundle replacement and target-version readiness. A macOS Electron journey covers headless promotion and persistent PTY identity. A disposable locally signed Electron canary exercised real ShipIt and a temporary LaunchAgent with the compiled production supervisor; full packaged Orca and Linux/Windows serve updates remain uncollected.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/8457", @@ -799,7 +712,11 @@ "protection": "partial", "owner": "editor-runtime", "layer": "renderer-electron-contract", - "surfaces": ["Agent Session History View Log", "Monaco external-content reconciliation", "renderer crash containment"], + "surfaces": [ + "Agent Session History View Log", + "Monaco external-content reconciliation", + "renderer crash containment" + ], "platforms": ["macos", "linux", "windows"], "providers": ["local"], "coveredPlatforms": ["macos"], @@ -836,7 +753,9 @@ }, { "file": "src/renderer/src/components/editor/monaco-content-sync.undo-history.test.ts", - "assertions": ["a real Monaco read-only live-tail append leaves canUndo false while an ordinary external update remains undoable"] + "assertions": [ + "a real Monaco read-only live-tail append leaves canUndo false while an ordinary external update remains undoable" + ] }, { "file": "src/renderer/src/components/editor/MonacoEditor.content-owner.test.tsx", @@ -844,15 +763,21 @@ }, { "file": "src/renderer/src/components/editor/EditorContent.monaco-lifecycle.test.tsx", - "assertions": ["a same-pane path switch unmounts the prior outer Monaco before real mount reconciliation refreshes the stale target; the prior retained model content and undo sentinel remain untouched"] + "assertions": [ + "a same-pane path switch unmounts the prior outer Monaco before real mount reconciliation refreshes the stale target; the prior retained model content and undo sentinel remain untouched" + ] }, { "file": "src/renderer/src/components/editor/monaco-content-sync.bench.ts", - "assertions": ["with forced GC and deterministic settlement between every arm, fresh real-Monaco 9 MiB and 50 MiB models alternate 30 append and 30 replacement samples after five warmups; append p95 stays below 50/100ms and at least 2x faster"] + "assertions": [ + "with forced GC and deterministic settlement between every arm, fresh real-Monaco 9 MiB and 50 MiB models alternate 30 append and 30 replacement samples after five warmups; append p95 stays below 50/100ms and at least 2x faster" + ] }, { "file": "src/renderer/src/components/editor/monaco-content-sync.undo-retention.bench.ts", - "assertions": ["five 10 MiB read-only live-tail appends retain zero Monaco undo-service and ArrayBuffer bytes while the undoable control retains at least 50 MiB"] + "assertions": [ + "five 10 MiB read-only live-tail appends retain zero Monaco undo-service and ArrayBuffer bytes while the undoable control retains at least 50 MiB" + ] }, { "file": "tests/e2e/agent-session-log-tail-stability.spec.ts", @@ -921,17 +846,27 @@ "summary": "The production View Log journey preserved viewport, selection, Find state, renderer liveness, and forced-GC/native budgets across three watcher appends while canUndo remained false." } ], - "runtimeBudget": { "p95Seconds": 600, "scope": "local focused renderer tests plus one Electron production-journey scenario" }, - "flakeHistory": { "status": "unknown", "evidence": "New deterministic gate with local macOS passes; CI soak history is not yet available." }, - "redGreenEvidence": { "status": "complete", "evidence": "A fail-first real-Monaco test observed canUndo=true after one read-only live-tail append, and the forced-GC 50 MiB control retained 104,858,630 bytes in Monaco's undo service. After the fix the read-only arm retained zero undo-service bytes while the editable control stayed undoable. The retained Electron gate also proves each fixed watcher arm preserves viewport, selection, Find state, and non-undoability. Production builds never install its legacy setValue control." }, - "performanceBudget": { "required": true, "evidence": "Every update retrieves the model value once and performs at most one equality-or-prefix comparison; a matching append submits only the suffix. The registered Node commands require --expose-gc and --pool=threads so worker GC is available. Current p95: 9 MiB append 5.93-7.12ms versus replacement 114.09-145.89ms; 50 MiB append 26.84-34.91ms versus replacement 602.36-699.21ms. The new 50 MiB retention arm measured 104,858,630 undo-service bytes and 104,857,790 ArrayBuffer bytes for the undoable control versus zero for read-only live-tail sync. Electron forced-GC JS-heap, renderer working-set, and OS-private-memory budgets also pass." }, + "runtimeBudget": { + "p95Seconds": 600, + "scope": "local focused renderer tests plus one Electron production-journey scenario" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "New deterministic gate with local macOS passes; CI soak history is not yet available." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "A fail-first real-Monaco test observed canUndo=true after one read-only live-tail append, and the forced-GC 50 MiB control retained 104,858,630 bytes in Monaco's undo service. After the fix the read-only arm retained zero undo-service bytes while the editable control stayed undoable. The retained Electron gate also proves each fixed watcher arm preserves viewport, selection, Find state, and non-undoability. Production builds never install its legacy setValue control." + }, + "performanceBudget": { + "required": true, + "evidence": "Every update retrieves the model value once and performs at most one equality-or-prefix comparison; a matching append submits only the suffix. The registered Node commands require --expose-gc and --pool=threads so worker GC is available. Current p95: 9 MiB append 5.93-7.12ms versus replacement 114.09-145.89ms; 50 MiB append 26.84-34.91ms versus replacement 602.36-699.21ms. The new 50 MiB retention arm measured 104,858,630 undo-service bytes and 104,857,790 ArrayBuffer bytes for the undoable control versus zero for read-only live-tail sync. Electron forced-GC JS-heap, renderer working-set, and OS-private-memory budgets also pass." + }, "promotionCriteria": [ "Collect stable soak history on macOS, Linux, and Windows.", "Accumulate 100 consecutive deterministic gate passes or 14 days without unexplained flakes." ], - "knownGaps": [ - "No live Windows or Linux View Log evidence yet." - ], + "knownGaps": ["No live Windows or Linux View Log evidence yet."], "demotionRule": "Quarantine the Electron scenario if it flakes without a product or harness bug; demote if viewport/Find drift, renderer loss, p95 regression, or memory retention exceeds the registered budgets." }, { @@ -941,23 +876,10 @@ "protection": "partial", "owner": "terminal-runtime", "layer": "renderer-unit", - "surfaces": [ - "terminal lifecycle", - "dead-session reconciliation", - "tab creation" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon" - ], - "coveredPlatforms": [ - "macos" - ], + "surfaces": ["terminal lifecycle", "dead-session reconciliation", "tab creation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the reconcile guards that exist on main@1282f5c2d. Broader targeted-hasPty resume paths, no-hot listing counts, and live Electron survival arrive with the pending reliability stack.", "motivatingLinks": [ @@ -1037,31 +959,12 @@ "Manage Sessions", "Resource Manager" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime", - "mobile-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime", "mobile-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "Local macOS deterministic evidence covers the renderer snapshot/coordinator, exact local and SSH-shaped PTY request settlement, active-last and pinned terminal-tab routing, component-unmount continuation, and the existing current/legacy daemon management contract. Windows Electron process absence, live SSH/WSL behavior, and remote-runtime/mobile host completion remain explicit gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/8001" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/8001"], "invariant": "Every terminal surface confirmed in the invoking renderer is force-closed exactly once after daemon management settles, later-created surfaces and non-terminal tabs survive, and exact shutdown requests are limited to deduplicated current non-runtime PTY bindings of the confirmed surfaces.", "oracle": "Snapshot terminal entity IDs before the first await; mutate ownership, active selection, bindings, and tab presence while daemon management is pending and between bounded close batches; then assert only the immutable targets disappear from both terminal stores, active targets close last with valid editor/browser/deactivated post-state, every captured exact PTY promise settles before callbacks, and no provider inventory sweep or late-tab kill occurs.", "commands": [ @@ -1164,26 +1067,10 @@ "hidden terminal parking", "agent resume authority" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh", - "runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh", "runtime"], "coverageNotes": "Live macOS Electron tests prove exact local PTY disappearance after parked-tab close and detached-pgid descendant death after agent close. Deterministic tests cover daemon and SSH routing, local/daemon pending-snapshot ownership across natural exit, stale-root descendant-signal suppression, graceful-to-immediate kill upgrades, duplicate-kill completion sharing, locale-stable bounded/fresh/coalesced process-table reads, deadline-safe successor scans, cycle-safe linear descendant traversal, target-only escalation indexing, 32-wide bulk teardown, source-scan timestamp identity, same-second PID ambiguity, ordinary runtime close ownership, unified-only hydration, split ownership, pane detach transfer, restart alias hydration, and late-hook suppression; live Linux, Windows, WSL, SSH, and remote-runtime process evidence remains pending.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/8628", @@ -1426,9 +1313,7 @@ }, { "file": "src/main/runtime/rpc/e2ee-channel-v2.test.ts", - "assertions": [ - "mobile E2EE v2 continues to reject additive runtime capability metadata" - ] + "assertions": ["mobile E2EE v2 continues to reject additive runtime capability metadata"] }, { "file": "src/main/runtime/rpc/runtime-client-capabilities.test.ts", @@ -1556,33 +1441,11 @@ "protection": "partial", "owner": "terminal-runtime", "layer": "shared-main-renderer-contract", - "surfaces": [ - "terminal lifecycle", - "tab creation", - "PTY spawn", - "startup cwd persistence" - ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "ssh", - "remote-runtime" - ], + "surfaces": ["terminal lifecycle", "tab creation", "PTY spawn", "startup cwd persistence"], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "ssh", "remote-runtime"], "coverageNotes": "Local macOS evidence covers the shared missing-dir fallback policy, main pty:spawn recovery and metadata, no-flag and reattach strictness, renderer IPC flag routing, SSH-tagged and remote-runtime omission, and the visibility-gated terminal notice. Daemon shares the same pre-provider main cwd decision but lacks a live daemon-provider run; WSL UNC paths are exempt from the probe by design and lack a live run; Linux/Windows and mobile/API strictness are gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/7239", @@ -1719,21 +1582,9 @@ "runtime reload and session replacement", "loopback restart and stall recovery" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS execution of the generated Pi/OMP extension plus the shared hook normalizer. WSL fallback behavior is covered with mocked native-fetch failure and Windows curl handoff. Daemon PTYs use the same generated extension without a distinct delivery path. SSH/relay ingest uses the shared normalizer, but no live remote Pi process is exercised.", "motivatingLinks": [ @@ -1823,21 +1674,9 @@ "provider session dedupe", "sidebar and mobile identity" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the ownership/dedupe suite on main@1282f5c2d. Queued/pending resume-claim indexing, same-session and wrong-session hook proofs, and Electron repeat-activation coverage arrive with the pending stack (#7008).", "motivatingLinks": [ @@ -1851,9 +1690,7 @@ "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/resume-sleeping-agent-session.test.ts" ], - "testFiles": [ - "src/renderer/src/lib/resume-sleeping-agent-session.test.ts" - ], + "testFiles": ["src/renderer/src/lib/resume-sleeping-agent-session.test.ts"], "assertionRefs": [ { "file": "src/renderer/src/lib/resume-sleeping-agent-session.test.ts", @@ -1922,28 +1759,10 @@ "terminal exit retirement and restart restore", "mixed-version fallback" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coverageNotes": "Deterministic macOS tests cover controller claims, daemon and SSH/relay operation replay, mixed-version selection, runtime ownership, exact provisional handoff, durable terminal retirement, two independent viewer mirrors, guarded adoption of legacy live PTYs, and completion classification when either the outer remote transport or authoritative host/provider process inspection becomes unreachable. The adoption harness models v1.4.150 agent/setup/shell tabs, current-generation restart and reconnect, exact handle/incarnation/worktree/host checks, topology CAS, competing clients, split-pane/group restoration, WSL ownership, and SSH owner rejection. The secondary parity repro runs independent clients against one headless remote Orca runtime over encrypted pairing and a real daemon-backed PTY, with tokened fixture-process identity separated from unrelated Codex app-server startup probes. The automated primary topology runs an isolated headed macOS Orca desktop server plus a separate paired web client and proves viewer-local fresh/resume focus, exact legacy placement, writable PTYs, unrelated-terminal survival, and host/client cleanup. SSH coverage is provider/relay contract and fault-injection coverage only; it does not substitute for paired-server coverage. Live Windows, Linux, WSL, SSH, and physical paired-Linux hosts remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/8878", @@ -2032,9 +1851,7 @@ }, { "file": "src/relay/pty-handler.test.ts", - "assertions": [ - "strict relay inspection rejects a missing PTY" - ] + "assertions": ["strict relay inspection rejects a missing PTY"] }, { "file": "tests/e2e/remote-agent-completion-authority.unit.test.ts", @@ -2315,31 +2132,12 @@ "browser and mobile handoff", "app restart" ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "local", - "remote-runtime", - "ssh", - "wsl" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "remote-runtime", - "ssh", - "wsl" - ], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["local", "remote-runtime", "ssh", "wsl"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "remote-runtime", "ssh", "wsl"], "coverageNotes": "Platform-neutral deterministic tests separate the durable Active Server preference from per-client connection, selected-workspace, browser-session, and execution-host routing. The composed regression models Local desktop -> connect/navigate Windows 2 -> reveal a local terminal -> restart. Multi-client browser host overrides, multi-server profile caches, generic settings IPC rejection, local and remote workspace ownership, and restart reset of transient routing are covered. Live desktop UI runs remain uncollected.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/9687" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/9687"], "invariant": "Only an explicit user change in Settings > Remote Orca Servers > Advanced > Active Server may mutate activeRuntimeEnvironmentId. Connecting, pairing, disconnecting, selecting or revealing a workspace or terminal, browser/mobile handoff, remote navigation, and reconnect must use transient or target-owner routing and must never rewrite the durable preference. Generic settings mutation cannot bypass the dedicated preference IPC.", "oracle": "Start with Active Server=Local desktop, connect and navigate Windows 2, then reveal a local terminal and assert it succeeds while the persisted preference remains local before and after restart. Repeat with multiple clients and servers, browser host switches, remote-owned and local-owned workspaces, pairing/connect/disconnect, and generic settings writes. Assert only the dedicated validated preference method changes activeRuntimeEnvironmentId and stale host-operation completions cannot overwrite the newly selected transient host.", "commands": [ @@ -2452,20 +2250,9 @@ "hidden-to-visible transitions", "window wake" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence on main@1282f5c2d, including #7192's runtime-mirror geometry authority slice. Deterministic provider-contract coverage now includes settled window-wake reassertion and SSH relay applied-size readback. Live shell-visible SSH/remote geometry and Windows ConPTY readback remain non-blocking gaps.", "motivatingLinks": [ @@ -2599,27 +2386,10 @@ "protection": "partial", "owner": "terminal-rendering", "layer": "renderer-unit", - "surfaces": [ - "terminal search", - "links", - "WebGL", - "decorations", - "keyboard navigation" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "surfaces": ["terminal search", "links", "WebGL", "decorations", "keyboard navigation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the WebGL/link/search containment suites on main@1282f5c2d, adopting the #6949 atlas-recovery rename and #7133's reveal hardening tests. Core addon-load throw containment and live typed-input survival arrive with #7004 and a live follow-up.", "motivatingLinks": [ @@ -2743,34 +2513,18 @@ "xterm scrollbar DOM", "scrollback" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Renderer-unit coverage proves the shared xterm DOM intent path. Live Electron evidence is PR validation evidence for local macOS only until the flow has stable automation; live SSH, WSL, Linux, and Windows paths remain unproved.", - "motivatingLinks": [ - "STA-1341" - ], + "motivatingLinks": ["STA-1341"], "invariant": "A user-driven xterm scrollbar thumb or track scroll updates the live terminal scroll intent before tab, visibility, or layout resume enforces intent, so resume preserves the latest dragged viewport instead of an older pinned line.", "oracle": "Pointerdown on .xterm-scrollbar or .xterm-slider followed by xterm viewport movement records the new pinned viewport, and enforcing current intent restores that dragged line instead of stale top intent.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts" ], - "testFiles": [ - "src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts" - ], + "testFiles": ["src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts"], "assertionRefs": [ { "file": "src/renderer/src/lib/pane-manager/terminal-scroll-intent.test.ts", @@ -2827,23 +2581,9 @@ "protection": "none", "owner": "startup-persistence", "layer": "upgrade-fixture", - "surfaces": [ - "startup", - "upgrade", - "session restore", - "daemon restore" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl" - ], + "surfaces": ["startup", "upgrade", "session restore", "daemon restore"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -2879,9 +2619,7 @@ "Run second restart after current code writes upgraded state.", "Record startup timing and failure artifact." ], - "knownGaps": [ - "No fixture corpus or command yet." - ], + "knownGaps": ["No fixture corpus or command yet."], "demotionRule": "Cannot promote without old production fixture provenance." }, { @@ -2900,13 +2638,8 @@ "resize", "exit cleanup" ], - "platforms": [ - "linux", - "macos" - ], - "providers": [ - "local" - ], + "platforms": ["linux", "macos"], + "providers": ["local"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The live Electron Playwright slice exists only on the pending reliability stack. It registers here with its owning split PR.", @@ -3057,20 +2790,10 @@ "CJK repaint", "cursor and resize" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], - "coveredPlatforms": [ - "windows" - ], - "coveredProviders": [ - "daemon" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], + "coveredPlatforms": ["windows"], + "coveredProviders": ["daemon"], "coverageNotes": "Issue #8048 now has deterministic wrapper and cold-restore re-anchor tests plus a Windows PR-CI harness that drives the built daemon through 25 real ConPTY workspace-close races while an unrelated witness PTY stays alive. Keyboard reset, CJK repaint, WSL, and full visible Electron coverage remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6541", @@ -3264,21 +2987,9 @@ "resize", "render" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Platform-neutral unit coverage proves the Resource Manager closed badge performs one readiness seed, coalesces unknown spawn signals, skips known-session reattach signals, and installs no interval. Broader terminal interaction coverage remains on the pending reliability stack.", "motivatingLinks": [ @@ -3314,9 +3025,7 @@ }, { "file": "src/main/ipc/pty.test.ts", - "assertions": [ - "global inventory starts local and SSH provider listings concurrently" - ] + "assertions": ["global inventory starts local and SSH provider listings concurrently"] }, { "file": "src/renderer/src/components/status-bar/resource-session-inventory.test.ts", @@ -3386,18 +3095,8 @@ "provider ownership", "diagnostics bundle" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The crash-breadcrumb recording and its test exist only on the pending reliability stack. It registers here with its owning split PR.", @@ -3452,44 +3151,105 @@ "main PTY batching", "runtime path provenance", "runtime terminal wait detection", + "SSH relay frame decoding", + "SSH PTY source retention", + "SSH PTY reconnect waves", "renderer ACK", "xterm scheduler", "hidden output" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [], - "coverageNotes": "Local macOS evidence covers the existing main-process pending-output caps plus deterministic runtime path-provenance history reuse and saturated-tail wait detection. Daemon stream write(false)/drain contracts, cross-session drain priority, and bounded queued tails arrive with the pending perf slice; live flood/latency artifacts remain gaps.", + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["ssh"], + "coverageNotes": "Local macOS evidence covers main-process pending-output and projection-admission caps, runtime path-provenance history reuse, saturated-tail wait detection, direct-SSH source retention/accounting, always-on V1 negotiation with legacy peer fallback, exact provider-generation pause ownership, renderer exit/data ordering, decoder input bounds, bounded reconnect scheduling, and WSL stdio transport settlement through deterministic unit contracts. A joined main/runtime oracle exercises renderer-sourced headed semantics and headless-model snapshot semantics without launching a live paired server. Separately, a macOS-hosted Docker OpenSSH run exercises only the deployed Linux relay and direct SSH provider with an exact 256 KiB source-credit plateau, concurrent PTY typing, fixed-size filesystem frames, Git churn, and owner-lease reconnect. Neither deterministic topology labels nor Docker SSH constitute live headed desktop, headless orca serve, or physical WSL evidence; local/daemon, prior-version daemons, Windows named pipes/ConPTY, folder workspaces, and mixed-version clients also remain uncovered.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6836", "https://github.com/stablyai/orca/pull/6858", "https://github.com/stablyai/orca/pull/7002", "https://github.com/stablyai/orca/pull/7054" ], - "invariant": "High-volume terminal output must stay bounded across daemon socket writes, main runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input.", - "oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, and ACK-gated in-flight bounds. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.", + "invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Every new SSH session offers V1 and every same-build relay supports it; legacy delivery is reachable only when capability negotiation proves the peer cannot use V1. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; provisional activation data cannot project, reconnect may release it only into the exact attempt's private recovery quarantine after contract validation, and exit seals that quarantine against later same-token frames until ordered admission or exact cancellation proof. Reconnect checkpoint capture freezes exact provider-generation/PTy admission, cancels queued old work, and waits for the running raw completion before exposing its accepted checkpoint; timeout or failure detaches only that PTy's old model, makes the checkpoint unavailable, and releases the bounded fence without closing the shared provider, while the same failure outside an active exact migration remains generation-fatal and overlapping reconnects preserve any earlier outstanding fence. Pending renderer projection IDs are capped and compacted across split remainders, exit preparation owns its renderer fence through finalization, exit-time cancellation transfers published projections before proof commit, generation close fences late proof while draining exact projection waiters, closed-generation identity compacts without weakening stale rejection, canceled source-delivery retirement retains at most one ordered token per PTY, and pause/resume targets only the exact provider generation. Additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.", + "oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, a 1,024-ID projection cap with split-remainder compaction, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, exact provider-generation pause ownership, renderer exit/data ordering, and ACK-gated in-flight bounds. The negotiation seam proves every initial connection and reconnect offers V1, the same-build relay advertises V1 without launch flags, old clients remain token-free, and method-not-found peers fall back without installing source ACK publication. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations without releasing in-flight byte credit. Replacement reservation rolls back every earlier span on a later failure, commit requires the exact frozen transfer states, and an authoritatively reclaimed covered span rejects commit without local trim authority while rollback removes the reservation idempotently. The exit-deadline oracle publishes a source-backed desktop projection, advances a fake clock to cancellation, and requires one projection transfer, proof commit, preparation, and final exit with no provider close, retained obligation, duplicate finalization, or process-lifetime cancellation tombstone; a second controlled proof remains pending across generation close and must never publish final exit, while both paths release the renderer preparation lease exactly once. The model-migration oracle accepts span A, blocks span B's raw emulator completion, begins disconnect migration, and proves no checkpoint or reconnect attach can remain at A; releasing B advances the exact checkpoint to B and projects B once. Its fake-clock arm reaches the 10-second deadline, requires checkpoint-unavailable, one exact model reset, zero retained admission charge and timers, and no effect from late raw settlement. Its two-PTY failure arm rejects one running raw callback under an active exact migration and requires one checkpoint-unavailable result, one model reset, zero provider closes, retained sibling checkpoint and transport liveness, zero charges and timers, and an unaffected other generation; a non-migrating sibling oracle still requires generation close. The intake additionally closes 2,048 sequential provider generations into one exact range, rejects stale events across that range, and preserves an out-of-order live gap until its own close. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, claim-gated projection under same-decoder-turn response/data delivery, exact early-ACK reservation through send settlement or same-token retry, recovery completion visible before lease-held frames transfers those frames only into the private quarantine and admits no output until exact checkpoint-to-recoveryEnd coverage, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved legacy-primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-pending-projection-admissions.test.ts src/main/ipc/ssh-pty-legacy-projection.test.ts src/main/ipc/ssh-pty-model-admission.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime-path-candidate-history.test.ts", - "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/orca-runtime-tail-wait-memo.test.ts" + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/orca-runtime-tail-wait-memo.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-model-admission.test.ts src/main/ipc/ssh-pty-output-model-migration.test.ts src/main/ssh/ssh-relay-session-model-migration.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/git-response-stream-ownership.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-recovery-interleavings.test.ts src/relay/relay-pty-source-recovery-completion.test.ts src/relay/relay-pty-source-restore-retry.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/dispatcher.test.ts src/relay/pty-handler-source-publication.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-legacy-projection.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-generation-guard.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot", + "pnpm exec vitest run src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/ssh-pty-consumer-session-adapter.test.ts src/main/persistence.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts src/relay/relay-pty-source-recovery-interleavings.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts --reporter=dot", + "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + // Historical evidence commands; the override is ignored after the always-on transition. + "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" ], "testFiles": [ "src/main/ipc/pty.test.ts", + "src/main/ipc/pty-pending-projection-admissions.test.ts", + "src/main/ipc/ssh-pty-legacy-projection.test.ts", "src/main/runtime/orca-runtime-path-candidate-history.test.ts", "src/main/runtime/orca-runtime.test.ts", - "src/main/runtime/orca-runtime-tail-wait-memo.test.ts" + "src/main/runtime/orca-runtime-tail-wait-memo.test.ts", + "src/main/providers/ssh-pty-provider.test.ts", + "src/main/providers/ssh-pty-notification-routing.test.ts", + "src/main/providers/ssh-pty-source-delivery-ledger.test.ts", + "src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts", + "src/main/providers/ssh-pty-provider-exit-race.test.ts", + "src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts", + "src/main/ssh/ssh-relay-session.test.ts", + "src/main/ssh/ssh-relay-session-data-delivery.test.ts", + "src/main/ssh/ssh-relay-session-model-migration.test.ts", + "src/main/ssh/ssh-relay-session-recovery-races.test.ts", + "src/main/ssh/ssh-relay-session-incarnation.test.ts", + "src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts", + "src/main/ssh/ssh-relay-session-terminal-error.test.ts", + "src/main/ssh/ssh-pty-recovery-retention-budget.test.ts", + "src/main/ssh/ssh-pty-retired-source-deliveries.test.ts", + "src/main/ssh/relay-protocol-backpressure.test.ts", + "src/relay/protocol-backpressure.test.ts", + "src/relay/git-response-stream-ownership.test.ts", + "src/relay/pty-handler.test.ts", + "src/relay/pty-source-credit-ledger.test.ts", + "src/relay/pty-source-credit-scheduler.test.ts", + "src/relay/relay-pty-source-publication.test.ts", + "src/relay/relay-pty-source-recovery-interleavings.test.ts", + "src/relay/relay-pty-source-recovery-completion.test.ts", + "src/relay/relay-pty-source-restore-retry.test.ts", + "src/relay/dispatcher.test.ts", + "src/relay/pty-handler-source-publication.test.ts", + "src/relay/ssh-pty-source-credit-adapter.test.ts", + "src/shared/pty-consumer-session.test.ts", + "src/main/ipc/ssh-pty-model-admission.test.ts", + "src/main/ipc/ssh-pty-output-intake.test.ts", + "src/main/ipc/ssh-pty-output-model-migration.test.ts", + "src/main/ipc/ssh-pty-output-generation-guard.test.ts", + "src/main/ipc/ssh-pty-output-exit-deadline.test.ts", + "src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts", + "src/main/runtime/rpc/terminal-source-range-ledger.test.ts", + "src/main/runtime/rpc/terminal-multiplex.test.ts", + "src/main/ssh/ssh-multiplexer-transport-writer.test.ts", + "src/main/ssh/ssh-channel-multiplexer.test.ts", + "src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts", + "src/main/ssh/ssh-relay-deploy.test.ts", + "src/main/persistence.test.ts", + "src/relay/dispatcher-client-writer.test.ts", + "src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts", + "tests/e2e/ssh-docker-relay-perf.spec.ts" ], "assertionRefs": [ { @@ -3497,7 +3257,22 @@ "assertions": [ "total renderer in-flight output is capped across many PTYs", "active PTY pending output is prioritized during renderer backpressure", - "combined pending output exceeding the interactive size limit is batched" + "combined pending output exceeding the interactive size limit is batched", + "reconnect replacement cannot redirect pause or resume away from the exact provider generation", + "data arriving while SSH exit waits for renderer projection settlement cannot follow exit preparation" + ] + }, + { + "file": "src/main/ipc/pty-pending-projection-admissions.test.ts", + "assertions": [ + "pending projection IDs retain exactly 1,024 entries then transfer the full ordered run", + "published prefixes compact before append while transfer state survives every split remainder" + ] + }, + { + "file": "src/main/ipc/ssh-pty-legacy-projection.test.ts", + "assertions": [ + "split projection publication remains pending until its exact display and accounting ranges are fully published" ] }, { @@ -3523,6 +3298,297 @@ "returns a blocked wait result for generic Codex interactive prompts", "resolves tui-idle when a stale Codex prompt is followed by the ready header" ] + }, + { + "file": "src/main/providers/ssh-pty-notification-routing.test.ts", + "assertions": [ + "provisional source frames remain unprojected and are discarded on rollback", + "held and later recovery frames route only to the private sink until final commit, without ordinary listeners or live PTY publication", + "exact source frames use their immutable incarnation without invoking or mutating the legacy incarnation resolver", + "exit during private recovery retires the activation ledger at final commit and rejects late same-token frames", + "private recovery retirement restores the exact predecessor without issuing a second cancellation", + "older rollback cancels only its exact token without replacing a newer activation" + ] + }, + { + "file": "src/main/providers/ssh-pty-source-delivery-ledger.test.ts", + "assertions": [ + "exit seals a provisional or private-recovery lease against later same-token admission", + "a stale transfer retains rollback authority and requests exact cancellation instead of orphaning its token" + ] + }, + { + "file": "src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts", + "assertions": [ + "same-decoder-turn source data waits for claim validation, failed claim rollback, and exact cancellation proof while a newer activation remains live" + ] + }, + { + "file": "src/main/providers/ssh-pty-provider-exit-race.test.ts", + "assertions": [ + "exit before a fresh spawn reply drops provisional source data and settles exact cancellation" + ] + }, + { + "file": "src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts", + "assertions": [ + "source-credit restore-required generic reattach fails as expired instead of returning an outputless live PTY" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts", + "assertions": [ + "fifty reattaches use at most eight workers and healthy siblings finish before one slow and one failed PTY", + "initial connection and automatic reconnect both offer source credit" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session.test.ts", + "assertions": [ + "each transient PTY failure is retried once without tearing down successful provider registration" + ] + }, + { + "file": "src/main/persistence.test.ts", + "assertions": [ + "retired per-target source-credit selections are removed during normalization" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session-model-migration.test.ts", + "assertions": [ + "overlapping reconnect attaches remain blocked behind the old-generation per-PTY model fence and receive only the post-completion exact checkpoint", + "stale-owner retry retains the old-generation fence through raw completion, then requests checkpoint-unavailable restore" + ] + }, + { + "file": "src/main/ssh/ssh-pty-recovery-retention-budget.test.ts", + "assertions": [ + "fragmented recovery is bounded per PTY and per session in source units, charged bytes, and frames", + "UTF-16 string storage and record overhead are charged before aggregate admission" + ] + }, + { + "file": "src/relay/pty-source-credit-ledger.test.ts", + "assertions": [ + "an exact early cumulative ACK reserves eligibility without advancing credit before send settlement and survives only an exact same-token retry", + "source retention charges max UTF-8 or UTF-16 storage plus one record", + "fragmented multi-PTY source, charged-byte, and frame caps bind independently" + ] + }, + { + "file": "src/relay/pty-source-credit-scheduler.test.ts", + "assertions": [ + "one rejecting delivery is removed after one reservation attempt while prior and later peer reservations remain committed without head-of-line blocking" + ] + }, + { + "file": "src/relay/git-response-stream-ownership.test.ts", + "assertions": [ + "an 8-byte encoded producer capacity emits exactly two 6-byte payload chunks and one end call while the first bulk write is saturated" + ] + }, + { + "file": "src/relay/pty-handler.test.ts", + "assertions": [ + "a fresh plain pending entry after transformed source-only output omits inherited rawLength and completes in exactly three admission calls" + ] + }, + { + "file": "src/relay/dispatcher.test.ts", + "assertions": [ + "a saturated legacy primary remains required backpressure and is never detached as an additional subscriber" + ] + }, + { + "file": "src/relay/pty-handler-source-publication.test.ts", + "assertions": [ + "immutable spawn and attach activation identity settles before the first source frame in the response/data interleaving", + "one saturated additional subscriber is bounded and detached without pausing the native PTY or negotiated owner" + ] + }, + { + "file": "src/relay/relay-pty-source-recovery-interleavings.test.ts", + "assertions": [ + "failed exit publication retains sealed delivery for late cumulative ACK and exact token recovery", + "exact owner recovery republishes exit without reopening source admission", + "a failed recovery-completion frame rolls back its fence and the next exact owner republishes it before live admission", + "failed activation response retains private retry authority, and exact retry clears activation only after successful settlement" + ] + }, + { + "file": "src/relay/relay-pty-source-recovery-completion.test.ts", + "assertions": [ + "unadmitted recovery completion retries once writer capacity returns, stays single-flight, and releases its bounded capacity listener" + ] + }, + { + "file": "src/relay/relay-pty-source-restore-retry.test.ts", + "assertions": [ + "invalid-checkpoint cancellation retains the exact delivery until restore response settlement, then retry mints a fresh activation and emits one live source frame" + ] + }, + { + "file": "src/main/ssh/relay-protocol-backpressure.test.ts", + "assertions": [ + "main SSH decoder accepts one maximum frame plus 1 MiB partial input and rejects one extra byte", + "a throwing continuation clears retained input, releases one pause epoch, and publishes one typed ownership error" + ] + }, + { + "file": "src/relay/protocol-backpressure.test.ts", + "assertions": [ + "relay decoder accepts one maximum frame plus 1 MiB partial input and rejects one extra byte", + "a throwing continuation clears retained input, releases one pause epoch, and publishes one typed ownership error" + ] + }, + { + "file": "src/main/ipc/ssh-pty-legacy-projection.test.ts", + "assertions": ["provider-generation close drains exact projection terminality waiters"] + }, + { + "file": "src/main/ipc/ssh-pty-model-admission.test.ts", + "assertions": [ + "disposal resumes every paused provider generation exactly once, including pause-only rejected entries", + "migration cancels queued old-generation work while retaining the one running raw completion", + "a callback failure outside migration closes its generation and rejects a sibling PTY admission" + ] + }, + { + "file": "src/main/ipc/ssh-pty-output-intake.test.ts", + "assertions": [ + "exit remains behind accepted model, desktop projection, and required remote obligations", + "exit timeout cancels only the matching delivery and keeps unrelated provider work usable", + "desktop source identity and scanner facts commit atomically or roll back without credit", + "renderer exit preparation remains owned through finalization and releases on duplicate, failure, or generation close", + "2,048 monotonic provider closes compact into one exact stale-generation range while out-of-order live gaps remain admissible" + ] + }, + { + "file": "src/main/ipc/ssh-pty-output-model-migration.test.ts", + "assertions": [ + "migration waits for blocked span B before exporting its checkpoint and projects B exactly once", + "migration timeout returns checkpoint-unavailable, resets one exact PTY model, and releases admission charge and timers before late raw settlement", + "migration-owned callback failure resets only the failed PTY while late settlement cannot advance it and sibling checkpoint, provider transport, and another generation remain live" + ] + }, + { + "file": "src/main/ipc/ssh-pty-output-generation-guard.test.ts", + "assertions": [ + "sequential generation closure compacts to one exact range without weakening stale rejection", + "out-of-order closures expose the exact unclosed generation count below the closed high-water" + ] + }, + { + "file": "src/main/ipc/ssh-pty-output-exit-deadline.test.ts", + "assertions": [ + "published projection transfer precedes cancellation-proof span reclamation with one preparation and final exit", + "generation close fences a pending cancellation proof from final exit and retained obligations" + ] + }, + { + "file": "src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts", + "assertions": [ + "replacement reserves only immutable spans whose retained model-sequence end is covered by the authoritative snapshot", + "renderer-sourced and headless-sourced replacement commits only after current-generation SnapshotEnd sequence coverage", + "partial replacement reservation rolls every prior exact transfer back to the live stream obligation", + "commit and rollback reject a transfer state replaced by concurrent mutation", + "a span reclaimed by cancellation proof rejects replacement commit without local trim authority, then exact rollback removes the reservation idempotently" + ] + }, + { + "file": "src/main/runtime/rpc/terminal-source-range-ledger.test.ts", + "assertions": [ + "partial cumulative byte ACK releases exact credit while retaining the immutable covering source frame", + "a contiguous higher-generation recovery token is admitted while the prior token remains unsettled", + "stale stream and source generations cannot advance or replace the current ledger identity" + ] + }, + { + "file": "src/main/runtime/rpc/terminal-multiplex.test.ts", + "assertions": [ + "blocked renderer and headless snapshot serialization admits one covered and one trailing source span", + "SnapshotEnd makes exactly the covered span upstream-ACK eligible while the trailing span is delivered and ACKed live", + "snapshot replacement, trailing replay, and exit complete without provider cancellation or retained obligation", + "a partial cumulative byte ACK does not settle its source frame or detach the stream before recovered-token output", + "a parsed stale-generation ACK cannot release in-flight byte credit or flush queued output" + ] + }, + { + "file": "src/shared/pty-consumer-session.test.ts", + "assertions": [ + "mismatched recovery is typed for one-shot fallback while the stale principal remains a subscriber and the retained principal and lease preserve recovery authority" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session-data-delivery.test.ts", + "assertions": [ + "the state-machine stale-owner error clears cached state and retries exactly once without resume", + "completion visible before lease-held recovery frames transfers those frames only into private quarantine and projects them once after exact checkpoint-to-recoveryEnd coverage and live handoff", + "invalid recovery cancels only its replacement token without physical PTY or ownership teardown" + ] + }, + { + "file": "src/main/ssh/ssh-relay-session-recovery-races.test.ts", + "assertions": [ + "empty recovery retains recoveryEndSu as the first live-frame continuity anchor", + "validated token cancellation drops queued late frames without physical PTY teardown", + "after recovery quarantine ownership transfer, rejected cancellation publication or proof closes only the exact provider generation and releases its provider, publishers, mux, and activation state once", + "negative, credited-ahead, checkpoint-mismatched, and under-covering recovery cancellation proofs fail closed against the highest privately observed range", + "overlapping recovery never cancels on or mutates the replacement mux, checkpoint, lease, provider state, or ownership" + ] + }, + { + "file": "src/main/ssh/ssh-pty-retired-source-deliveries.test.ts", + "assertions": [ + "10,000 ordered token cancellations for one PTY retain one latest-token retirement record", + "the next activation or PTY exit boundary clears only that PTY's retirement record" + ] + }, + { + "file": "src/main/ssh/ssh-multiplexer-transport-writer.test.ts", + "assertions": [ + "write(false) owns one frame and later ordinary traffic waits for drain", + "queued control is selected before ordinary PTY backlog at drain while each lane remains FIFO", + "four-control fairness guarantees ordinary progress without starving control", + "one coalesced liveness bypass is allowed per saturated epoch", + "saturation and drain transitions are reported exactly once" + ] + }, + { + "file": "src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts", + "assertions": [ + "source ACK, cancellation, exit, request, and response control frames precede queued pasted PTY input after drain", + "control and ordinary frames preserve lane FIFO while ordinary input progresses after four control writes" + ] + }, + { + "file": "src/main/ssh/ssh-channel-multiplexer.test.ts", + "assertions": [ + "self-imposed writer saturation suppresses false death and rebases both health clocks on drain" + ] + }, + { + "file": "src/relay/dispatcher-client-writer.test.ts", + "assertions": [ + "encoded producer frames stay below high-water-minus-reserve capacity", + "one fixed-size filesystem compatibility frame is admitted only on an empty sink", + "control remains reserved while producer retention stays bounded" + ] + }, + { + "file": "src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts", + "assertions": [ + "WSL stdin forwards write(false), callback settlement, and drain without claiming a live WSL topology" + ] + }, + { + "file": "tests/e2e/ssh-docker-relay-perf.spec.ts", + "assertions": [ + "a stalled renderer ACK produces an exact 256 KiB negotiated source-credit plateau while a second SSH PTY remains responsive", + "fixed-size filesystem frames and Git churn complete without stream corruption while active typing remains within budget", + "the negotiated owner lease reconnects and the existing SSH workspace terminal remains usable" + ] } ], "evidenceRuns": [ @@ -3552,6 +3618,87 @@ "result": "passed", "durationSeconds": 21.33, "summary": "2 test files passed, 609 tests passed on pushed commit 4fb14eac3897." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 2.24, + "summary": "Sixteen deterministic SSH relay/session/source/decoder files passed 192 tests, including exit-sealed private recovery, retained stale-transfer cancellation authority, exact private-frame proof watermarks, one retirement record across 10,000 same-PTY token rotations, stale-owner fallback, and scheduler rejection isolation; no live topology was exercised." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/git-response-stream-ownership.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 11.81, + "summary": "Two deterministic relay stream and PTY batching files passed 120 tests, including exact encoded chunk capacity and transformed-to-plain metadata isolation." + }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 7.17, + "summary": "Three deterministic main/runtime files passed 78 tests, including joined renderer/headless snapshot admission, exact partial cumulative credit, and recovered-token continuity; no live paired server or Docker topology was exercised." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 6.89, + "summary": "Thirteen current-head deterministic files passed 150 tests across exit-sealed private recovery, stale-transfer cancellation authority, bounded latest-token retirement, restore retirement, fail-closed recovery cancellation, partial ACK and token rotation, reclaimed-span replacement, mux lane fairness, provisional activation, and exit-proof ordering; no live topology was exercised." + }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 1.86, + "summary": "Seven current-worktree provider/session files passed 94 tests after a red deterministic seam reproduced completion visibility before lease-held recovery data; the candidate transfers held and later frames only into private recovery quarantine, final-commits after fence/admission, retires exited activation state, and preserves fail-closed malformed, gapped, overlapping, missing-body, cancellation-proof, and replacement behavior." + }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts src/relay/relay-pty-source-recovery-interleavings.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 1.63, + "summary": "Five deterministic provider, relay, deploy, and session files passed 70 tests after red seams proved that source frames mutated legacy incarnation state and generic restore-required reattach returned an outputless live PTY. The candidate keeps exact source identity side-effect free, fails generic restore-required reattach as expired, offers V1 through reconnect, and proves a failed recovery activation stays private and exact-retryable until successful response settlement; no live topology was exercised." + }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 60, + "summary": "All four deployed Linux relay/direct-SSH cases passed after an exact-current E2E rebuild at code commit 5611bb45b51a/tree 8e85c3afedec: main bundle SHA-256 cbaf4e997d74bbe0ae1179bc20e52b122c60f0b56ee49ca607e12ae4125a4342 and Linux-x64 relay SHA-256 a7438fc47c4da0223ceaafab621086e14b6cdee427f53a3bf2e710fa99bcf2e6. Direct typing was 3.7/109.1 ms median/worst, ACK-stalled typing was 5.1/109.7 ms at exactly 262,144 held source units, fixed-size filesystem/Git churn was 144.9/153.2 ms with 104 bulk reads, and owner-lease reconnect completed in 15.8 seconds." + }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 60, + "summary": "Four post-rebase Docker OpenSSH/deployed-relay tests passed: direct typing median/worst 107.7/113.6 ms, stalled-ACK typing 3.6/107.3 ms at an exact 262,144-source-unit plateau, fixed-size filesystem/Git churn 148.1/161.1 ms with 93 bulk reads, and terminal owner-lease reconnect in 15.7 seconds." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 56.7, + "summary": "All four always-on deployed Linux relay/direct-SSH cases passed after rebuilding at current-main merge commit adba3410fe4427ceb7525f3fdce2ec58973263a7/tree e43d87340666ca3a733bdb38007c24f95be3f219: main bundle SHA-256 3c304ffc0618520e42bede9a52f72d4b7bb68cbfe543974f89e82a3c477c44b3 and Linux-x64 relay SHA-256 366cb7ccf2e4b388cc6f81a8f6055ab9fde57d2d91381767005c15908832e776. Direct typing was 5.3/108.9 ms median/worst, ACK-stalled typing was 4.8/107.6 ms at exactly 262,144 held source units, fixed-size filesystem/Git churn was 152.0/173.3 ms with 84 bulk reads, and owner-lease reconnect completed in 14.3 seconds; no paired-runtime topology was exercised." } ], "runtimeBudget": { @@ -3560,15 +3707,15 @@ }, "flakeHistory": { "status": "unknown", - "evidence": "Focused main-process backlog tests are deterministic unit slices. The combined perf soak still needs runtime history before promotion." + "evidence": "Focused main-process backlog and direct-SSH reconnect/memory/decoder tests are deterministic unit slices. On 2026-07-27 the source-credit reconnect case passed once in the full four-test Docker suite (15.7 seconds) and three consecutive isolated repeat-each runs (21.1, 19.8, and 15.3 seconds). On 2026-07-28 it passed against private recovery transfer in 17.7 seconds alone and 16.0 seconds in the full four-case suite; the combined perf soak still needs longer runtime history before promotion." }, "redGreenEvidence": { "status": "partial", - "evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output is capped per PTY and in total, total-pressure trimming prefers background pending output before active pending output, trimmed pending tails preserve seq/rawLength metadata, and ACK-gated in-flight output remains bounded. Intentionally restoring the old path-history shape failed the focused scale test after 774ms and 4,096 replacements; disabling the ordinary-tail branch failed with a retained 253,999-character waitText. The fixed shapes passed their focused suites. Existing renderer tests cover replay/backlog slices. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion." + "evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output plus projection IDs stay within exact caps, direct-SSH source/recovery memory and both frame decoders stay within exact caps, and 50-PTY reconnect isolates slow/failing siblings. On pre-fix 4bf54da9b, reconnect pause targeted the replacement provider, disposal omitted a pause-only generation, normal and generation-close exit paths never released the renderer preparation lease, and the projection-cap seam did not exist; the same oracles pass with exact generation indexing, paused-generation enumeration, barrier-owned exit leases, and the 1,024-ID transfer latch. The deterministic main-to-relay drain oracle failed with ordinary-2 and ordinary-3 ahead of a later control frame under the single FIFO, then passed with control-first lane selection and four-write ordinary fairness. On exact pre-fix baseline db167ea3d with test-only oracles, cancellation proof reclaimed the published projection span before transfer and generation close allowed a late proof to publish final exit; both deterministic oracles pass after proof acquisition and commit are separated under the exact deadline barrier. Before snapshot admission, the trailing span entered transferring instead of remaining open; after retaining model-sequence ends and reserving at the serialized fence, the joined main/runtime oracle passed for renderer and headless sources with upstream boundaries 4 then 8. Before cumulative partial ACK and source rotation, the ledger rejected ACK 40 of 100 and the multiplex seam detached instead of publishing recovered token B; both exact oracles pass while stale stream and source generations remain rejected. Before restore-record retirement, invalid-checkpoint recovery followed by restore/retry returned restoreRequired a second time instead of opening a fresh activation; the same deterministic oracle passes after exact response-settlement cleanup. With test-only oracles on pre-fix 4bf54da9b, an existing-owner mismatch lacked the stale-owner code and both shared authority and main fallback rows failed; the candidate types that refusal while stale fresh admission remains a subscriber and the retained principal and lease can still recover. Before the recovery-quarantine activation fix, completion was visible at source unit 4 while lease-held recovery ended at 8, so the fence rejected and cancellation failed; the same byte-identical seam now transfers held ranges only into private quarantine, admits output after exact body/fence validation, and the deployed Linux relay reconnect passes. Other controls previously failed with serial reattach, UTF-8-only accounting, and roughly 32 MiB decoder retention. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion." }, "performanceBudget": { "required": true, - "evidence": "Suggested ceilings: renderer in-flight <=8MB total, <=512KB per PTY plus active reserve, renderer queued chars <=2MB, dropped backlogs 0, hidden restore <=1000ms, active key median/worst <=75ms/300ms in perf scenarios. With 1,024 retained provenance candidates, 4,096 pathless chunks dropped from 78.84ms and 4,096 array replacements to 1.74ms and zero replacements. Brace-free 1 KiB output dropped from 20.32/79.12/318.23ms to 0.31/0.64/2.35ms across 4,096/16,384/65,536 chunks. Repeated ordinary 252,000-character tail checks dropped from 96.34/464.20/941.79/3,796.70ms to 3.21/15.68/31.34/129.72ms across 100/500/1,000/4,096 updates." + "evidence": "Suggested ceilings: renderer in-flight <=8MB total, <=512KB per PTY plus active reserve, renderer queued chars <=2MB, dropped backlogs 0, hidden restore <=1000ms, active key median/worst <=75ms/300ms in perf scenarios. Always-on capability offers add no persisted setting reads, polling, scans, subprocesses, provider fanout, or reconnect-path reevaluation. Restore-record retirement adds one exact map-identity check at response settlement with no polling, timers, scans, subprocesses, or provider calls. The Docker SSH gate enforces median/worst typing below 500/2,000 ms; the current private-transfer artifact observed 5.5/16.3 ms during an exact 256 KiB stalled-credit plateau, 141.2/151.5 ms during 95 completed filesystem/Git bulk reads, and a 16.0-second reconnect. With 1,024 retained provenance candidates, 4,096 pathless chunks dropped from 78.84ms and 4,096 array replacements to 1.74ms and zero replacements. Brace-free 1 KiB output dropped from 20.32/79.12/318.23ms to 0.31/0.64/2.35ms across 4,096/16,384/65,536 chunks. Repeated ordinary 252,000-character tail checks dropped from 96.34/464.20/941.79/3,796.70ms to 3.21/15.68/31.34/129.72ms across 100/500/1,000/4,096 updates." }, "promotionCriteria": [ "Split daemon stream backpressure into a deterministic provider/IPC contract if full E2E is flaky.", @@ -3576,7 +3723,10 @@ "Keep stress variants non-blocking until stable runtime history exists." ], "knownGaps": [ - "Daemon stream write(false)/drain/cleanup and bounded queued-tail contracts are not registered in the current command.", + "Local daemon/provider and prior-version daemon topologies were not executed.", + "Headed paired desktop, headless orca serve, folder workspace, and mixed-version client topologies were not executed.", + "Physical WSL and Windows named-pipe/ConPTY topologies were unavailable and were not executed.", + "Ubuntu 20.04/glibc 2.31 packaging was not physically executed; cross-target relay bundling is build evidence only.", "Runtime provenance coverage is deterministic and does not include a live high-throughput provider artifact.", "Terminal-wait scale evidence is deterministic and does not yet include a live saturated-tail event-loop artifact.", "Current command does not prove renderer parse pressure, scheduler queue depth, event-loop delay, or active key latency.", @@ -3598,15 +3748,8 @@ "provider ownership", "startup restore" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "daemon", - "local" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["daemon", "local"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The fail-closed degraded-daemon hardening and its contracts exist only on the pending reliability stack. It registers here with its owning split PR.", @@ -3662,24 +3805,10 @@ "runtime terminal stop", "restored terminal teardown" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "A real daemon server over a local socket proves a fresh adapter can kill a live session before any prior client operation. Main-process tests prove renderer IPC, runtime kill, and runtime exact-stop wait for the provider swap and issue zero shutdowns to the fallback provider, while SSH spawn and kill bypass the local barrier. The same shared logic runs on Linux and Windows; live platform runs remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/7742", @@ -3777,25 +3906,13 @@ "terminal create idempotency", "provider listing", "reattach", + "provider reattach incarnation fencing", "unknown liveness" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "ssh", - "remote-runtime", - "wsl" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["ssh", "remote-runtime", "wsl"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["ssh", "remote-runtime"], "coverageNotes": "Deterministic renderer coverage proves startup publishes the state returned by ssh.connect, retained native and runtime SSH payloads are admitted through production routes only with valid complete authority, stale cleanup cannot unregister a replacement runtime terminal, direct SSH Git and folder panes clear and retry by exact authority, one authority chain stops after two automatic attempts even when each timeout exceeds the rolling window, rejected acknowledgements mutate no store maps, and one shared exact attempt admits every concurrent split-pane spawn and reattach while preserving the first PTY as the tab fallback. A later sibling failure rotates the tab once, stale callbacks from the prior attempt mutate no state, split remount activity suppression is counted per leaf, primary PTY exit promotes a bound survivor or preserves an empty continuation gap for a late sibling, and primary, non-primary, or null-PTY detach preserves exact authority on both resulting tabs. Intentional pane disposal cancels its settlement timer without breaking StrictMode remount timeout ownership. Target snapshot hydration/reconnect preserves sibling SSH/local/WSL/runtime state, and a mounted remote-runtime terminal survives repeated transport partitions without changing PTY identity. Direct SSH coordinator tests cover immediate terminal finalization, hydration correction, damping, bounded retry, and telemetry non-interference. Client/server heartbeat tests cover timer suspension, socket generations fence stale callbacks, cold restored-terminal attachment retries, cached pixels remain unhealthy until authoritative replay, automatic retries stop after one minute, manual reconnect preserves the PTY, and pane closure releases recovery UI state. Current macOS Electron journeys against an ephemeral Linux Docker SSH target cover exact-authority repo/worktree hydration, live terminal recovery after disconnect/reconnect, and eager six-terminal remount after renderer reload. A Windows remote-runtime smoke covers reachability and PTY round-trip. Multi-target live fanout, paired-close, WSL, and patched live partition journeys remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6951", @@ -3804,9 +3921,10 @@ "https://github.com/stablyai/orca/pull/7009", "https://github.com/stablyai/orca/pull/8597" ], - "invariant": "SSH, WSL, and remote-runtime restore paths must treat provider listing failures and unknown liveness as unknown, not dead, while still avoiding duplicate spawn and clearing expired relay leases exactly once. Direct SSH reconnect must atomically clear only exact-target live PTY bindings, preserve relay identity, retry Git and folder panes without paired close or provider shutdown, and allow at most two automatic attempts in one authority chain even when each settlement exceeds the rolling window. A rejected acknowledgement mutates no store map. A successful exact split-pane spawn or reattach must retain that attempt as shared live authority until sibling leaves settle; the first success cannot consume sibling authority, a sibling failure can start at most one second tab-wide attempt, and prior-attempt callbacks become inert after rotation. Once the retry budget is exhausted, a failure cannot start attempt three or revoke attempt-two authority from siblings that may still settle. Primary PTY exit must promote a bound survivor or preserve exact authority through an empty activation gap, and split detach must project that authority to both resulting tabs. Hydrated PTY hints cannot supersede a current exact-attempt owner, and target snapshot hydration/reconnect cannot reset sibling SSH, local, WSL, or runtime-owned state. Every restored remote terminal must preserve its provider PTY identity. After a recoverable partition the same authenticated runtime must reattach the same PTY, reject detached input, apply the latest viewport, and report healthy only after authoritative replay. Automatic PTY recovery stops after one bounded minute without a fatal terminal error; a manual reconnect starts a newly fenced epoch against the same PTY, and closed panes retain no recovery UI state. One capability-gated terminal-create mutation must produce at most one host PTY across an unknown response outcome, remain manually retryable after cutoff, and never let a stale completion replace a newer pane lifecycle.", - "oracle": "Deterministic tests cover bounded stale-handle replacement, suspended heartbeat clocks, cold and established subscription failure, ten partition/recovery cycles, automatic-recovery cutoff, manual reconnect, and exact direct SSH binding recovery. They assert one atomic store publication clears only exact-target PTY indexes, null-PTY activation remains unchanged, relay identity survives, Git and folder panes retry symmetrically, another target/local/WSL/runtime panes remain byte-identical through target snapshot hydration and reconnect, only an accepted exact failure or timeout starts the second attempt, two 31-second timeouts cannot start a third settlement-triggered attempt, rejected stale/mismatched acknowledgements preserve every store map, and concurrent split-pane spawn and reattach callbacks both commit under the same attempt ID after the first success replaces pending state with live shared authority. A sibling failure revokes that shared authority and starts exactly one second attempt; duplicate failures and late first-attempt PTY callbacks preserve the second attempt and every state map. Attempt-two failure retains continuation authority for later siblings, primary exit promotes a bound survivor or preserves the lease until a late sibling binds, and primary plus non-primary detach retain exact authority and history on both resulting tabs. Both remount callbacks consume split-count activity suppression, intentional dispose emits no failure/timeout, and a same-attempt StrictMode remount still owns one timeout. Hydration clears an untrusted PTY hint without clearing its current pending owner, healthy current-authority bindings suppress correction, hydration finalizes once, and reconnect emits no paired close lifecycle. Tests also assert one unsubscribe per remote-runtime epoch, observable recovery phases, stable PTY identity, resumed snapshot/output/input, no healthy state before replay, no retry or input after cutoff, a new manual epoch against the same PTY, quiet recovery UI with an explicit Reconnect action, pane-close state cleanup, one stable create mutation id, old-runtime no-retry behavior, cross-process PTY adoption, and bounded in-flight coordination.", + "invariant": "SSH, WSL, and remote-runtime restore paths must treat provider listing failures and unknown liveness as unknown, not dead, while still avoiding duplicate spawn and clearing expired relay leases exactly once. Direct SSH reconnect must atomically clear only exact-target live PTY bindings, preserve relay identity, retry Git and folder panes without paired close or provider shutdown, and allow at most two automatic attempts in one authority chain even when each settlement exceeds the rolling window. A rejected acknowledgement mutates no store map. A successful exact split-pane spawn or reattach must retain that attempt as shared live authority until sibling leaves settle; the first success cannot consume sibling authority, a sibling failure can start at most one second tab-wide attempt, and prior-attempt callbacks become inert after rotation. Once the retry budget is exhausted, a failure cannot start attempt three or revoke attempt-two authority from siblings that may still settle. Primary PTY exit must promote a bound survivor or preserve exact authority through an empty activation gap, and split detach must project that authority to both resulting tabs. Hydrated PTY hints cannot supersede a current exact-attempt owner, and target snapshot hydration/reconnect cannot reset sibling SSH, local, WSL, or runtime-owned state. Every restored remote terminal must preserve its provider PTY identity, including the authoritative incarnation returned by a successful session-ID reattach. After a recoverable partition the same authenticated runtime must reattach the same PTY, reject detached input, apply the latest viewport, and report healthy only after authoritative replay. Automatic PTY recovery stops after one bounded minute without a fatal terminal error; a manual reconnect starts a newly fenced epoch against the same PTY, and closed panes retain no recovery UI state. One capability-gated terminal-create mutation must produce at most one host PTY across an unknown response outcome, remain manually retryable after cutoff, and never let a stale completion replace a newer pane lifecycle.", + "oracle": "Deterministic tests cover bounded stale-handle replacement, suspended heartbeat clocks, cold and established subscription failure, ten partition/recovery cycles, automatic-recovery cutoff, manual reconnect, and exact direct SSH binding recovery. They assert one atomic store publication clears only exact-target PTY indexes, null-PTY activation remains unchanged, relay identity survives, Git and folder panes retry symmetrically, another target/local/WSL/runtime panes remain byte-identical through target snapshot hydration and reconnect, only an accepted exact failure or timeout starts the second attempt, two 31-second timeouts cannot start a third settlement-triggered attempt, rejected stale/mismatched acknowledgements preserve every store map, and concurrent split-pane spawn and reattach callbacks both commit under the same attempt ID after the first success replaces pending state with live shared authority. A sibling failure revokes that shared authority and starts exactly one second attempt; duplicate failures and late first-attempt PTY callbacks preserve the second attempt and every state map. Attempt-two failure retains continuation authority for later siblings, primary exit promotes a bound survivor or preserves the lease until a late sibling binds, and primary plus non-primary detach retain exact authority and history on both resulting tabs. Both remount callbacks consume split-count activity suppression, intentional dispose emits no failure/timeout, and a same-attempt StrictMode remount still owns one timeout. Hydration clears an untrusted PTY hint without clearing its current pending owner, healthy current-authority bindings suppress correction, hydration finalizes once, and reconnect emits no paired close lifecycle. A provider-level session-ID reattach returns an incarnation, then a legacy exit without an incarnation must resolve to that returned identity rather than minting a fallback identity. Tests also assert one unsubscribe per remote-runtime epoch, observable recovery phases, stable PTY identity, resumed snapshot/output/input, no healthy state before replay, no retry or input after cutoff, a new manual epoch against the same PTY, quiet recovery UI with an explicit Reconnect action, pane-close state cleanup, one stable create mutation id, old-runtime no-retry behavior, cross-process PTY adoption, and bounded in-flight coordination.", "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/startup/ssh-startup-reconnect.test.ts src/renderer/src/lib/resolved-worktree-execution-host.test.ts src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts src/renderer/src/runtime/web-session-terminal-handle-events.test.ts src/renderer/src/store/slices/terminal-pty-identity-replacement.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts", @@ -3818,6 +3936,7 @@ "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" ], "testFiles": [ + "src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts", "src/renderer/src/startup/ssh-startup-reconnect.test.ts", "src/renderer/src/lib/resolved-worktree-execution-host.test.ts", "src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts", @@ -3995,6 +4114,12 @@ "a detached null-PTY leaf remains marked for pending activation before ownership transfer" ] }, + { + "file": "src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts", + "assertions": [ + "a successful session-ID reattach remembers its returned incarnation before a later legacy exit is published" + ] + }, { "file": "src/main/ipc/repos-remote.test.ts", "assertions": [ @@ -4274,12 +4399,8 @@ "provider liveness", "restore" ], - "platforms": [ - "windows" - ], - "providers": [ - "wsl" - ], + "platforms": ["windows"], + "providers": ["wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -4337,17 +4458,8 @@ "metadata-only replay", "WebGL recovery" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The replay FIFO/burst coalescing product change and its tests exist only on the pending reliability stack; main still uses a single pendingReplayData slot. It registers here with its owning split PR.", @@ -4406,21 +4518,9 @@ "keyboard bypass", "JIS yen" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos", - "linux" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos", "linux"], "coveredProviders": [], "coverageNotes": "Local macOS and containerized Linux evidence, deterministic renderer-unit coverage for the Linux/Sogou candidate-key policy including the legacy orphaned-keyup fallback, and Electron/CDP live-PTY Sogou-style repros. Real Linux/Sogou OS IME automation, Windows ConPTY post-agent reset, and the CJK/Vietnamese/Arabic matrix remain registered gaps.", "motivatingLinks": [ @@ -4463,9 +4563,7 @@ }, { "file": "src/renderer/src/components/terminal-pane/terminal-paste-runtime.test.ts", - "assertions": [ - "paste/runtime forwarding avoids duplicate terminal payloads" - ] + "assertions": ["paste/runtime forwarding avoids duplicate terminal payloads"] }, { "file": "src/renderer/src/components/terminal-pane/terminal-ime-composition-tracker.test.ts", @@ -4602,14 +4700,8 @@ "TUI exit", "standard key input" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -4666,21 +4758,9 @@ "WSL", "remote runtime" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Deterministic main/renderer tests run on macOS and exercise simulated Windows plus POSIX/Linux process-confirmation behavior. They cover exact local-ConPTY membership, detached-child rejection, SSH and paired-runtime host-platform routing including legacy runtime PTY IDs, active-PTY ownership after worktree host changes, Windows-to-WSL routing, unknown-metadata fallback, production handler composition, KKP authorization on every host, process-confirmed Droid routing and launch-triggered confirmation, typed-alias and forged-OSC isolation, shells without OSC 133, split/detach ownership with preserved shell classification, stale PTY exit/rebind rejection, command-generation revocation, daemon v21 warm-reattach identity, unavailable inspection, and lazy subprocess/RPC callback counts. A live linux-arm64 Docker target additionally proved real relay deployment, SSH PTY host detection, and exact inactive/active KKP bytes. The Electron byte test runs only on Windows; live Windows evidence remains uncollected. Test failures report exact expected bytes; no product telemetry or raw terminal logging is added.", "motivatingLinks": [ @@ -4941,29 +5021,12 @@ "PTY identity", "foreground process confirmation" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon"], "coverageNotes": "Runtime and provider-contract tests on macOS cover exact-PTY authorization, local/daemon fresh confirmation, exact ConPTY membership, and unsupported-provider fail-closed behavior. Physical Windows, live Linux, SSH, WSL, and remote-runtime validation remain explicit gaps; providers without confirmation preserve conservative refusal on a shell conflict.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/8303" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/8303"], "invariant": "A guarded note send writes only to the exact PTY binding checked by the guard and only while permission/wait evidence allows input; fresh hook state conflicting with an ordinary shell foreground requires fresh provider confirmation of a recognized agent in that PTY.", "oracle": "Fresh explicit state plus ordinary PowerShell plus confirmed recognized agent is sendable on the same PTY. Confirmed shell/non-agent, unavailable confirmation, PTY exit, handle rebind, or a callback PTY mismatch returns a refusal or not-writable result and writes zero bytes.", "commands": [ @@ -5015,9 +5078,7 @@ }, { "file": "src/main/providers/local-pty-provider.test.ts", - "assertions": [ - "fresh confirmation is discarded when its owning local PTY exits" - ] + "assertions": ["fresh confirmation is discarded when its owning local PTY exits"] }, { "file": "src/main/providers/windows-conpty-process-membership.test.ts", @@ -5040,9 +5101,7 @@ }, { "file": "src/renderer/src/lib/active-agent-note-send.test.ts", - "assertions": [ - "selected active-agent note sends retain guarded paste and submit routing" - ] + "assertions": ["selected active-agent note sends retain guarded paste and submit routing"] }, { "file": "src/renderer/src/components/browser-pane/BrowserAnnotationSendMenuContent.test.tsx", @@ -5106,27 +5165,12 @@ "PTY writes", "bracketed paste" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local"], "coverageNotes": "Local macOS evidence covers the runtime PTY write contract and a live dev-runtime CLI repro. SSH, daemon, remote-runtime, Linux, and Windows remain provider/platform gaps; the product path stays provider-owned and does not add local filesystem or process assumptions.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/issues/7226" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/issues/7226"], "invariant": "Injected orchestration task prompts for recognized agent CLIs must send the prompt body inside one bracketed-paste frame, sanitize embedded ESC bytes, preserve chunk boundaries without losing the frame, and send Enter only after the paste frame completes.", "oracle": "Runtime tests assert the exact PTY write sequence and failure cleanup; orchestration tests assert dispatch/coordinator use the agent prompt path; the live CLI harness dispatches a 32KB task to a fake Codex-like TUI and requires marker present, bracketed paste present, zero unframed line breaks, and submit observed.", "commands": [ @@ -5246,30 +5290,12 @@ "app restart with retained daemon PTYs", "legacy update resume fencing" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "daemon", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh"], "coverageNotes": "Deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, the SSH in-process CLI fallback, and federated non-reveal. Two isolated macOS Electron journeys launch fake Codex workers through the real RPC path and record append-only spawn/interruption ledgers. They assert immediate inactive presentation, one live agent PID, stable PTY/incarnation/tab/leaf/worktree/Task/Dispatch identity, and no interruption after workspace re-entry; the restart journey additionally removes renderer ownership, marks the Dispatch legacy, retains the daemon process across an app restart, and proves exact background adoption with readable ACK output and no resume replay. Distinct A/B artifacts plus live SSH, WSL, folder, remote-runtime, Linux, and Windows cutover journeys remain explicit gaps.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/11107#discussion_r3663321387" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/11107#discussion_r3663321387"], "invariant": "Starting a worker in the coordinator's current workspace must materialize one inactive terminal tab before worker-start returns, preserve coordinator focus, and remain exactly once after workspace re-entry. After an app update or restart, an exact live legacy worker must fence automatic provider resume, adopt its original PTY into its original background pane, retain readable output, and clear the resume record without spawning, writing, signalling, interrupting, replacing, or focusing the worker. An exact existing target workspace must receive a discoverable tab without stealing coordinator focus; if renderer reveal fails, worker-start must expose that the live worker remains background-only. Run and Dispatch checks must resolve through the caller's stable pane identity when a terminal handle is reminted, while a live handle outranks mismatched pane metadata, explicit legacy terminal inspection remains handle-scoped, and remote or headless worker presentation remains background-only.", "oracle": "Drive Run create, Task create, and worker-start through production Electron runtimes with a deterministic Codex fixture. Require append-only ledgers with one still-live PID and no interruption, a visible inactive worker tab while the coordinator stays active, Run delivery through stable pane identity, and stable PTY/incarnation, tab, leaf, worktree, Task, and Dispatch across workspace re-entry. In a restart journey, retain the original daemon PTY and PID, remove renderer ownership, retain sleeping-session evidence, mark the Dispatch legacy, relaunch, and require exact inactive tab adoption, readable ACK output, cleared resume state, one spawn, and no resume argv or Conversation interrupted text after another workspace round trip. Unit tests separately assert authority-specific legacy affordances, exact identity and owner matching, retained-output fallback, pane-stable routing, federated non-activation, and SSH fallback parity.", "commands": [ @@ -5342,9 +5368,7 @@ }, { "file": "src/main/runtime/rpc/methods/orchestration-federation.test.ts", - "assertions": [ - "federated worker placement explicitly sets activate=false" - ] + "assertions": ["federated worker placement explicitly sets activate=false"] }, { "file": "src/main/ssh/ssh-remote-orca-cli.test.ts", @@ -5459,14 +5483,8 @@ "cursor repaint", "rewrite output" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -5502,10 +5520,7 @@ "Keep stress cases non-blocking until Windows runtime history is stable.", "Fail promotion on silent Windows environment skips." ], - "knownGaps": [ - "No manifest command yet.", - "No Windows CJK/emoji repaint command is wired." - ], + "knownGaps": ["No manifest command yet.", "No Windows CJK/emoji repaint command is wired."], "demotionRule": "Cannot promote if the oracle is screenshot-only or environment-skipped." }, { @@ -5523,14 +5538,8 @@ "local provider", "daemon provider" ], - "platforms": [ - "windows" - ], - "providers": [ - "local", - "daemon", - "wsl" - ], + "platforms": ["windows"], + "providers": ["local", "daemon", "wsl"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -5585,17 +5594,8 @@ "renderer CPU", "resize churn" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", @@ -5643,21 +5643,9 @@ "protection": "none", "owner": "terminal-performance", "layer": "daemon-provider-contract", - "surfaces": [ - "daemon stream", - "socket write", - "drain", - "hidden output", - "input starvation" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "daemon" - ], + "surfaces": ["daemon stream", "socket write", "drain", "hidden output", "input starvation"], + "platforms": ["macos", "linux", "windows"], + "providers": ["daemon"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The daemon batcher write(false)/drain contracts exist only on the pending reliability stack. It registers here with its owning split PR.", @@ -5715,23 +5703,12 @@ "workspace switch", "agent status" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap on main. The boot-hydration counters and their tests exist only on the pending reliability stack. It registers here with its owning split PR.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/7002" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/7002"], "invariant": "Terminal typing, focus, resize, tab/workspace switch, and agent/session restore must not trigger unbounded store projection, git status, provider listing, or per-pane polling work.", "oracle": "The current executable slice instruments boot-time local PTY registry hydration with repo counts, local-vs-remote repo skips, worktree enumeration counts, adapter/session listing counts, registration/skipped-session counts, duration, and failure phase. The broader oracle still needs instrumentation that counts store selector recomputes, git status requests, provider listings, and session scans during scripted hot interactions with many worktrees and terminal panes.", "commands": [], @@ -5780,20 +5757,9 @@ "metadata-only replay", "clear semantics" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over the merged #7133/#7173 restore and hidden-output ordering tests on main@1282f5c2d. The dirty-state exactness contract, normal-buffer clear semantics, and metadata-only replay remain pending-stack work.", "motivatingLinks": [ @@ -5808,9 +5774,7 @@ "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts" ], - "testFiles": [ - "src/renderer/src/components/terminal-pane/pty-connection.test.ts" - ], + "testFiles": ["src/renderer/src/components/terminal-pane/pty-connection.test.ts"], "assertionRefs": [ { "file": "src/renderer/src/components/terminal-pane/pty-connection.test.ts", @@ -5878,23 +5842,12 @@ "renderer stream", "terminal colors" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; no executable coverage is wired yet.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/6949" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/6949"], "invariant": "Startup OSC 10/11 color queries are answered out of band at startup only, never leak into shell/provider output streams, and ordinary runtime OSC color queries remain renderer-handled.", "oracle": "A provider-contract fixture records startup query replies, shell-visible bytes, renderer-visible bytes, and later runtime OSC behavior to prove no query leakage or color deadlock.", "commands": [], @@ -5922,10 +5875,7 @@ "Cover startup-only and runtime OSC paths separately.", "Keep screenshot evidence diagnostic only." ], - "knownGaps": [ - "No manifest command yet.", - "No startup color-query contract is wired." - ], + "knownGaps": ["No manifest command yet.", "No startup color-query contract is wired."], "demotionRule": "Cannot promote if success is based only on absence of visible artifacts." }, { @@ -5942,21 +5892,9 @@ "mobile subscription replay", "multi-mobile input floor" ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Deterministic local tests execute the exact injected mobile replay/generation gate, the React Native query classifier, stale-subscription sender, server-side single-responder election, query-reply RPC semantics, and live-output capture during async mobile fit. The provider write path is shared, but no live iOS/Android, SSH, WSL, or multi-device run is registered yet.", "motivatingLinks": [ @@ -6088,21 +6026,9 @@ "JSON subscribe fallback", "snapshot buffering" ], - "platforms": [ - "macos", - "linux", - "windows", - "mobile" - ], - "providers": [ - "remote-runtime", - "ssh", - "local", - "daemon" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows", "mobile"], + "providers": ["remote-runtime", "ssh", "local", "daemon"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence covers runtime-RPC stream budgets plus paired-renderer parse/discard credit. Deferred credit is shared by local and remote transports, batches ACKs at 192 KiB or 4 ms, grows per-stream windows from 512 KiB to 2 MiB and aggregate windows from 2 MiB to 8 MiB, bounds queued output to 256 KiB per stream, and caps each multiplex connection at 32 active or pending streams for an 8 MiB aggregate pending-output ceiling. Deterministic tests cover replay ordering, stale generations, malformed frames, hidden panes, queue eviction, disposal, send/recovery failure, repeated pending-slot replacement, reconnect, and round-robin fairness. The opt-in benchmark covers 1/20/100 ms RTT and 1/4/8 viewers, exact protocol-frame allocations, scheduler CPU, and measured @xterm/headless parser CPU/retained heap. Live Android restore evidence, browser/WebGL parser measurements, and legacy JSON subscribe parity remain required.", "motivatingLinks": [ @@ -6270,20 +6196,9 @@ "unicode width", "parser configuration" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS evidence over #7148's width-parity oracle on main@1282f5c2d. A broader byte corpus, a shared parser-construction assertion over terminal-unicode-provider.ts/pane-terminal-options.ts, and recorded agent-session corpora remain gaps.", "motivatingLinks": [ @@ -6295,9 +6210,7 @@ "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/headless-emulator-unicode-width.test.ts" ], - "testFiles": [ - "src/main/daemon/headless-emulator-unicode-width.test.ts" - ], + "testFiles": ["src/main/daemon/headless-emulator-unicode-width.test.ts"], "assertionRefs": [ { "file": "src/main/daemon/headless-emulator-unicode-width.test.ts", @@ -6354,23 +6267,9 @@ "protection": "none", "owner": "terminal-rendering", "layer": "renderer-observability", - "surfaces": [ - "hidden-output restore", - "snapshot replay", - "anomaly breadcrumbs", - "telemetry" - ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "remote-runtime" - ], + "surfaces": ["hidden-output restore", "snapshot replay", "anomaly breadcrumbs", "telemetry"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; the anomaly-breadcrumb machinery exists in this branch but no convergence probe is implemented.", @@ -6495,15 +6394,8 @@ "window wake", "render model" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon"], "coveredPlatforms": [], "coveredProviders": [], "coverageNotes": "Registered gap only; the live repro harness with a refresh-repair oracle exists but is not productized into the release-blocking terminal-rendering-golden suite.", @@ -6557,18 +6449,9 @@ "renderer reload / crash recovery", "surviving daemon and local PTYs" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon" - ], - "coveredPlatforms": [ - "macos" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon"], + "coveredPlatforms": ["macos"], "coveredProviders": [], "coverageNotes": "Local macOS unit evidence over the shared main-process delivery pipeline in registerPtyHandlers. The oracle drives a local mock PTY, but the in-flight/pending accounting is provider-agnostic and daemon PTYs ride the identical pipeline (the manual dev repro that proved the freeze was daemon-backed). SSH is a separate relay/credit path and never enters these counters; mobile/relay ride unaffected paths; WSL terminals ride this same local/daemon pipeline and are covered by the same accounting (no dedicated provider-contract test yet). Motivated by a production incident on v1.4.131 (three frozen/broken terminal panes in one desktop session) diagnosed to leaked in-flight/pending accounting across a renderer reload (rendererGraphEpoch 3). Live validation of the fix on a rebuilt dev instance surfaced a second leak mode (the boot window): after the reset ran, main resumed flushing a flooding PTY into the still-booting page before its pty:data listener re-registered, so those sends were dropped yet counted and re-pinned rendererInFlightChars at 524288. The gate now also covers holding sends until the renderer's pty:rendererDispatcherReady handshake. A watchdog self-heals a lost handshake: on each reset a one-shot ~10s timer arms and, if the handshake never arrives, force-opens the gate (rendererDispatcherReadyForcedCount increments) so a dropped handshake degrades to pre-handshake behavior instead of a permanent hold; the real handshake or a re-registration cancels it. Two new diagnostics — rendererPtyDispatcherReady and rendererDispatcherReadyForcedCount — expose the otherwise-invisible boot-window hold, which early-returns before ackGatedFlushSkipCount increments.", "motivatingLinks": [ @@ -6662,24 +6545,10 @@ "desktop filesystem watcher", "SSH relay filesystem watcher and live PTYs" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "remote-runtime", - "ssh" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "remote-runtime", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "remote-runtime", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "remote-runtime", "ssh"], "coverageNotes": "Deterministic tests cover one shared healthy child per runtime process plus at most four bounded fault-quarantine children, native desktop, paired-runtime, and WSL snapshot processes sharing the same eight-physical-child reservation with typed serialized event-driven capacity recovery, including recursive re-wait when crash recovery reclaims an announced slot, one quarantine attempt per watch lifetime, generation-scoped cancellation, bounded termination with removable deadline waiters, replacement-crawl errors that cannot be blessed by a late readiness ack, pre-ready paired-web cancellation registered before an unbounded capacity wait, already-resolved and late physical-exit retry of a rejected paired-web teardown, physical desktop-install and local/SSH PTY teardown, stale-generation rejection, per-child stat bounds, final-overflow RPC delivery, and renderer eviction before terminal callbacks can retry. Destructive local and SSH removal fences both ID-derived and resolved-cwd terminal roots, including sibling-root/cwd combinations with reverse admission rollback, closes descendant relay watches before parent deletion, enumerates authoritative provider/cwd ownership, falls back to daemon spawn cwd before OSC 7, and keeps daemon/relay immediate shutdown pending until native exit or a bounded fail-closed error; a dead relay PID discovered during attach also settles concurrent shutdown before its stale entry is reaped, while Windows runtime watcher deadlines retain one removable physical-close waiter, treat Node's error-close path as positive physical-exit proof, and clear root ownership on late close. The relay policy test maps a standard repository's base, Git common directory, and worktree roots to one healthy supervisor, then proves a shared-child failure recovers those roots in separate quarantine supervisors with overflow and resumed events. Abortable capacity, quarantine, runtime-root, relay, SSH, and relay pre-install setup waits attach one reaction to each shared promise and explicitly remove 10,000 cancelled caller closures while one anchor remains. Built-entry macOS harnesses kill the desktop/runtime and SSH relay watcher children, then require automatic resubscription and later events while the host process and relay PTY survive. SSH bundles the child boundary beside relay.js, requires both artifacts for install completeness, and preserves registration-owned same-root cancellation.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/5308", @@ -7154,9 +7023,7 @@ }, { "file": "config/scripts/package-electron-runtime-contract.test.mjs", - "assertions": [ - "every release platform packages and gates the hashed watcher child" - ] + "assertions": ["every release platform packages and gates the hashed watcher child"] }, { "file": "config/scripts/runtime-file-watcher-resource-probe.mjs", @@ -7470,30 +7337,12 @@ "provider terminal inventory", "relay capability enforcement" ], - "platforms": [ - "macos", - "linux", - "windows" - ], - "providers": [ - "local", - "daemon", - "ssh", - "wsl", - "remote-runtime", - "mobile-relay" - ], - "coveredPlatforms": [ - "macos" - ], - "coveredProviders": [ - "local", - "ssh" - ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime", "mobile-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "ssh"], "coverageNotes": "Deterministic macOS contract evidence covers opaque local- and SSH-shaped terminal ids, one bounded worktree listing, mismatch rejection, and the main/relay host-call adapter matrix. It does not launch a live PTY or provision a relay-hosted plugin.", - "motivatingLinks": [ - "https://github.com/stablyai/orca/pull/8549" - ], + "motivatingLinks": ["https://github.com/stablyai/orca/pull/8549"], "invariant": "terminal.sendText accepts only an explicit provider-owned terminal id present in one bounded inventory of the worktree resolved immediately before the send; an absent id causes zero send calls, and relay callers cannot supply their own capability grants or transport classification.", "oracle": "Resolve the active worktree once, list that worktree with the v0 terminal cap once, and assert zero sendTerminal calls for a mismatched opaque id versus exactly one send for matching local- and SSH-shaped ids; then run the same permission and schema cases through desktop-main and registered relay panel/worker adapters and compare error codes.", "commands": [ diff --git a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md new file mode 100644 index 00000000000..abaa7d108cb --- /dev/null +++ b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md @@ -0,0 +1,2949 @@ +# SSH Relay PTY Backpressure + +Date: 2026-07-27 + +Status: architecture gate closed and exact SSH V1 wire/API implemented +always-on for new SSH sessions; integrated lifecycle fixes reconciled at +the current working tree; live topology gaps remain explicit below + +Historical unbounded baseline: `badf91101babf96fa09cb79a8294f7e23b9f081c` +(the implementation branch parent). The implementation was rebased onto +`origin/main@d9fec8fd61d0349ae8cdf0eea594de7425db94c9`; final GitHub CI remains +the merge gate. + +## Scope + +This implemented design bounds SSH relay PTY output from the native PTY through the relay +dispatcher, SSH channel or relay socket, Electron main process, and renderer +parser. It covers: + +- application-level PTY output credit; +- relay stdout and socket drain handling; +- per-client isolation, fairness, ordering, and cleanup; +- bounded frame decoding on both sides of the relay protocol; +- memory budgets, diagnostics, compatibility, tests, and deployment. + +The sidebar reconnect fix in `9d3ae3adc7` is out of scope. The non-ancestor +commit `1500a92904` is design input only; it must not be cherry-picked. This +work does not change PTY input semantics, terminal-model interpretation, or +file/Git payload semantics. It does change replay transport, SSH producer +pause wiring, and bulk frame admission where they share the dispatcher sink. + +## Implemented architecture and evidence boundary + +The architecture review approved one shared semantic +`PtyConsumerSession` state machine with an SSH-specific transport adapter, +not a universal wire protocol. The exact SSH adapter contract is now +implemented. In this PR: + +- `PtyConsumerSession` owns authenticated client/owner generations, lease + recovery, capability intersection, and grant publication commit/rollback. +- `SshPtyConsumerSessionAdapter` binds those semantics to the existing + authenticated relay connection and the first framed `pty.openClient` + request. The relay dispatcher remains the only sink-publication authority. +- The relay source ledger owns immutable source spans, outstanding source + credit, token rotation, sealed exit, and cancellation proof. The dispatcher + writer owns byte admission, priorities, `write(false)`, callbacks, and drain. +- Spawn and attach responses carry immutable `sourceActivation` metadata. The + main mux installs a provisional receive lease synchronously in + `beforeResolve`; adjacent source notifications remain private to that lease + until spawn validation commits or a validated reconnect contract transfers + them into its private recovery quarantine. Rollback drops untransferred data + and waits for exact token-cancellation proof. +- `SshRelaySession` owns grant/reconnect state, bounded recovery quarantine, + exact recovery fencing, and cumulative ACK publication through the main SSH + mux. Failed recovery-cancellation publication or proof closes the exact + provider generation and its publishers once. +- The main SSH intake owns one atomic admission across the model, desktop + projection, and required remote consumers. Model completion and consumer + terminality make ACKs eligible; only the mux write callback publishes them. +- Desktop and remote replacement transactions carry immutable span identity. + Reserve happens before publication, commit requires the exact + generation/sequence fence, and failure, stale publication, or already + reclaimed spans reject commit and roll back idempotently without credit. +- Relay `restoreRequired` cancellation retains its exact delivery record until + the metadata response settles, then retires only that record so a retry can + mint a fresh token. Main exit timeout obtains cancellation proof before + transferring projections and committing reclamation, with generation checks + around every finalization step. +- The main-to-relay writer has three FIFO lanes: liveness, control, and + ordinary. Drain reselects liveness/control ahead of queued ordinary traffic, + while one ordinary frame is guaranteed after four control writes. + +The implementation is intentionally scoped: + +| Surface | State in this PR | Evidence | +| ------------------------------------------- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Same-build direct SSH/deployed relay, V1 | implemented; every session offers and relay advertises V1 | current-working-tree deterministic contracts plus provenance-bound macOS-hosted Docker OpenSSH | +| Capability/method-not-found legacy fallback | implemented bounded legacy compatibility | deterministic negotiation and writer contracts | +| Reconnect/owner recovery | implemented | exact contiguous recovery, fail-closed cancellation, eight-wide reattach tests, Docker reconnect | +| Headed/headless remote consumers | source-range rotation/replacement implemented | deterministic runtime/main seam only; no live paired-runtime claim | +| WSL stdio and Windows named pipe/ConPTY | shared transport paths preserved | deterministic/common-contract evidence only; no physical run | +| Local provider and local daemon | unchanged | outside SSH V1 negotiation; existing behavior only | +| Folder workspace | no `.git` dependency added | code-path review only; no dedicated live fixture | +| Prior-version/mixed-version peers | fail-closed or version-scoped legacy behavior preserved | deterministic negotiation/deploy contracts; no live old binary | +| Ubuntu 20.04/glibc 2.31 | no native dependency added | cross-target relay build only; no physical packaging run | + +Docker SSH proves the Linux SSH provider and deployed-relay topology. It does +not prove headed or headless paired-runtime behavior, WSL, Windows ConPTY or +named pipes, local daemon/provider behavior, folder workspaces, prior-version +processes, mixed-version clients, or the Ubuntu 20.04 packaging floor. + +The final exact-code Docker rerun rebuilt the E2E main bundle and all relay +targets from the current-main merge head. It is bound to implementation commit +`adba3410fe4427ceb7525f3fdce2ec58973263a7`, tree +`e43d87340666ca3a733bdb38007c24f95be3f219`, main bundle SHA-256 +`3c304ffc0618520e42bede9a52f72d4b7bb68cbfe543974f89e82a3c477c44b3`, +and deployed Linux-x64 relay SHA-256 +`366cb7ccf2e4b388cc6f81a8f6055ab9fde57d2d91381767005c15908832e776`. +No additional live topology proof is inferred from that run. + +## Verified baseline and migration boundary + +On the historical baseline, the SSH path was not protected by the main pending-data bound: + +1. `node-pty` calls `PtyHandler`'s `onData`. `PtyHandler` keeps the existing + 100 Ki-source-unit replay tail. The first ordinary flush waits 8 ms, a + sustained flush continues after 1 ms, and interactive or transformed output + may publish immediately; a turn publishes at most two 16 Ki-unit chunks. +2. `RelayDispatcher.notify('pty.data')` broadcasts each publication. The + ordinary notification path ignores `process.stdout.write()` and + `Socket.write()` returning `false`, so every later frame can extend a Node + writable queue after the first saturated write. +3. Main's `FrameDecoder.feed()` drains all complete frames synchronously. + `MultiplexerTransport` exposes neither read pause/resume nor write drain. +4. `SshRelaySession.wireUpPtyEvents` is the production SSH delivery owner. It + calls `runtime.onPtyData()` and then sends `pty:data` directly to the + renderer. It does not enter the `PtyPendingDataDrainQueue` installed by + `src/main/ipc/pty.ts` for local and daemon providers. +5. Main therefore has no SSH queue record against which to calculate useful + upstream progress. The current cumulative renderer ACK is converted to a + delta, the SSH path commonly emits zero, and the relay's + `PtyHandler` ignores `pty.ackData` in any case. +6. Production POSIX and Windows deployment launches the relay detached, then + attaches the real desktop bridge through a Unix socket or named pipe. + Dispatcher construction therefore cannot identify the usable session owner. +7. `OrcaRuntime.onPtyData` returns synchronously while headless-emulator writes + append captured strings to an unmetered asynchronous `writeChain`. +8. Relay fs/Git chunks use `notifyBulk`/producer chains outside any stated PTY + reserve; they can occupy the same ordered sink ahead of cancel, ACK, and + interactive output. +9. `computeRemoteRelayDir` content-hash-scopes both daemon files and endpoints. + Mixed deployed binaries are fenced, while a prior-version daemon can remain + alive in its old directory after upgrade. +10. `SshPtyProvider` omits the existing producer-pause hooks, and production + constructs `MultiplexerTransport` in both SSH sentinel handling and the WSL + hook path. +11. Lossless remote terminal streams ACK encoded byte deltas today and retain + only aggregate in-flight bytes; no existing record maps those bytes back to + immutable SSH source intervals. `694363805` extracted their UTF-16 + code-unit-preserving chunker to + `src/main/runtime/rpc/terminal-output-frame-chunks.ts`; it improves + performance and equivalence coverage but adds no provider/token identity. +12. `1fd0f731f` routes SSH folder-workspace automation launches to their owning + host. It does not change PTY output delivery or make folder workspaces Git + worktrees. +13. `077561f89` moved remote terminal stream UTF-8 measurement into + `terminal-stream-byte-length.ts`, preserving legacy flush boundaries and + partial over-limit counts. Those encoded-byte counters remain transport + budgets, not SSH source credit. +14. `2dac0741b` preserves ordered DEC mode 2031 subscribe/withdraw decisions + when the current main pending-data queue drops or salvages renderer output. + Its bounded cross-chunk scan state is projection metadata that a unified + SSH intake must preserve; it does not settle source spans. +15. `d547e278f` adds epoch-scoped, post-delivery watermarks for mobile + notification catch-up. That notification replay protocol is distinct from + mobile terminal streaming; its sequence and epoch cannot identify or settle + PTY source ranges. + +This project first moves ownership of every SSH provider data event out of the +direct `wireUpPtyEvents` send and into one main delivery intake. That intake +performs runtime ingestion, desktop delivery policy, remote-consumer +fan-out, and upstream span settlement exactly once. Adding another listener is +forbidden because it would double-ingest and double-render output. + +The initial 8 ms/continuation 1 ms cadence and two-write limit are scheduling +controls, not bounds. A continuous producer can currently grow the relay +writable queue, SSH buffers, decoder input, and Electron heap. The frame-header +`ack` remains transport liveness bookkeeping; it is not PTY credit and never +enters this ledger. + +The falsifiable invariant is: accepted SSH relay PTY output remains bounded +across relay writable queues, SSH/socket transport, main decoding and model +admission, renderer projection, and remote consumers while every source span +settles exactly once in order. The observable failure on this baseline is that +ordinary relay PTY notifications continue after `write(false)`, main drains all +complete frames synchronously, SSH bypasses `PtyPendingDataDrainQueue`, and +headless emulator writes enter an unmetered Promise chain. A sustained SSH +producer can therefore increase relay, transport, and main memory without a +finite upstream credit owner. + +Authority and delivery boundaries on this baseline are: + +```text +renderer/viewer + -> IPC or runtime RPC + -> Electron main / OrcaRuntime + -> SshRelaySession + SshPtyProvider + -> SshChannelMultiplexer + -> SSH channel / --connect bridge + -> RelayDispatcher + -> PtyHandler + -> node-pty / ConPTY +``` + +`PtyHandler` owns native output and replay; relay dispatcher/adapters own +client writes; `SshRelaySession.wireUpPtyEvents` owns production SSH ingestion; +`OrcaRuntime` owns the headless model; `src/main/ipc/pty.ts` owns +main-to-renderer accounting; and `terminal.multiplex` owns remote-consumer +delivery. The current bug is fragmentation across those owners, not a missing +renderer-only cap. + +## Required invariants + +1. Protocol credit uses one canonical unit: UTF-16 code units in the + pre-transform source span, called source units (`su`). Retained memory and + wire queues use exact bytes and never masquerade as source credit. +2. For a negotiated token, + `0 <= sentEnd - creditedEnd <= windowSu`; each admitted slice fits the + remaining window, so no frame overshoot is permitted. ACKs are cumulative, + monotonic, client/PTY/token scoped, and never exceed `sentEnd`. +3. Source-credit spans are immutable and stored in a cumulative ledger. + Queue data may merge, split, salvage, thin, or coalesce without moving, + copying, or destroying ledger boundaries. +4. A sink `write(false)` accepts its frame exactly once. No ordinary frame is + written again before drain. PTY admission stops before a reserved + control/liveness capacity is consumed. +5. A spawn/attach token remains `activating` until its metadata-only response + crosses the relay sink fence. Main may receive source frames after its + synchronous provisional install, but they remain lease-private and cannot + project. A validated reconnect contract transfers that lease into the + attempt's private recovery quarantine; final commit, model, and desktop + admission remain fenced until all recovery `pty.data` and the exact + completion fence agree. +6. Data is ordered within a token. `pty.exit` follows every accepted data frame + and cannot bypass the sink gate or activation fence. Publishing exit seals + new data but does not retire an uncredited suffix. +7. Every open token-owned span and consumer obligation ends exactly once in + `settled`, `transferred`, or `canceled`; a transfer may pass through the + non-terminal `transferring` state only while its named replacement fence is + outstanding. A canceled `restoreRequired` delivery remains addressable until + its exact response settles and retires only if the map still names that + record. Records are never merely abandoned while their relay token can live. +8. Desktop, mobile, web, and agent-session consumers follow the explicit + settlement policy below. A stalled recoverable desktop projection cannot + freeze a healthy lossless remote view. +9. One slow additional subscriber cannot retain unbounded data or stop a + healthy subscriber. The negotiated session owner is not torn down for one + PTY's backlog; constructor position never grants that role. +10. When every delivery for a PTY is blocked, pause the native PTY. Resume only + when both local and relay-wide low-water predicates hold. +11. An unexpected client loss transfers outstanding output to a bounded + reconnect-grace owner. Normal subscriber absence retains only the existing + replay tail and no live queue. +12. Decoder work, decoded bytes, model admission, and activation-hold turns are + bounded. A self-imposed read or local-write pause rebases both liveness + clocks. Main-to-relay drain reselects liveness, then control, ahead of + ordinary PTY input; four consecutive control selections force one ordinary + selection when both lanes remain non-empty. +13. Disconnect, provider replacement, renderer reload, exit, disposal, and + workspace removal have explicit bounded cleanup. After their required + publication, transfer, cancellation, or generation-close proof, they leave + no open span, token, writer callback, drain waiter, timer, cursor, or paused + PTY. Recovery-cancellation publication failure or rejected proof fails + closed by closing only the owning provider generation and disposing its + mux, provider, publishers, transferred activation state, and registration + once. +14. All counters are finite safe integers. A transformed frame requires a + valid `rawLength`; malformed, excessive, stale, and cross-client values + cannot create credit or crash either process. +15. The session owner is elected by an authenticated session grant and + identified by an owner generation. It is never inferred from dispatcher + construction, stdout, socket order, a path, `.git`, or a worktree. +16. Recovery and serialization control responses contain metadata only. + Source-ranged recovery or snapshot bodies use bounded producer lanes and + explicit completion fences before live delivery. +17. The required main model has bounded asynchronous admission. Relay credit + cannot advance while data waits outside that charged admission or after an + emulator failure. +18. Negotiated V1 has one upstream ACK owner. Renderer projection progress and + remote encoded-byte ACKs settle ledger obligations but never emit legacy + SSH ACK deltas. +19. Required obligations becoming terminal, cumulative ACK queueing, and ACK + publication are three monotonic states. A write callback publishes already + eligible credit; it never creates eligibility. +20. Desktop admission carries immutable span identity through model reserve, + projection queue, renderer send, ACK, salvage, reload, and replacement. + Failure either rolls back an uncommitted transaction or transfers a + committed obligation with proof. Exit timeout acquires cancellation proof + without mutation, transfers published projections, commits proof + reclamation, prepares/finalizes exit once, and rechecks the provider + generation before each post-proof mutation. +21. Projection drop, thinning, salvage, restore, and replay preserve current + main's ordered terminal side-effect facts and bounded scanner state, + including DEC mode 2031 subscribe/withdraw decisions. Pre-commit rollback + restores the prior scanner snapshot, committed transfer moves projection + state exactly once, and an explicit source gap resets cross-chunk state; + none of these facts creates source credit. +22. The immutable delivery identity is exactly `(id, providerGeneration, +clientGeneration, ownerGeneration, ptyIncarnation, deliveryToken)`. Each + admitted source span additionally fixes `spanId`, source/display half-open + ranges, transform metadata, and model-sequence end where required. +23. Receive activation begins at `checkpointSourceEndSu`. Every recovery + `pty.data` frame is contiguous from that cursor through `recoveryEndSu`; + only the exact `pty.recoveryComplete` writer settlement opens live output. +24. A reentrant cumulative ACK may reserve only the exact pending send + boundary for the same delivery identity. Failed send settlement retains + that boundary and permits only an exact same-token retry. +25. A fixed `fs.streamChunk` keeps the protocol's 256 KiB payload and offset + semantics. It may use the fixed-frame lane only from an empty, + unsaturated producer epoch; ordinary PTY and reshapable bulk frames still + obey the non-reserved capacity. +26. A lossless remote stream may advance its cumulative encoded-byte ACK to an + interior byte offset. Exact byte budget is released immediately, while the + covering immutable source frame remains unsettled until its complete + encoded boundary is crossed. +27. A lossless remote stream accepts an in-place source-token rotation only + when PTY ID, provider generation, and incarnation match; both client and + owner generations strictly increase; the token changes; and source/display + coordinates remain contiguous. Old or partially increasing identities are + rejected. +28. Snapshot replacement commits are idempotent and never synthesize byte or + source credit. Duplicate/closed ledger commits are no-ops; if cancellation + proof already reclaimed a reserved span, source replacement commit rejects, + rollback skips the absent span, and settle/detach prune cached mappings. +29. Exact exit seals a provisional or private-recovery receive lease before + any later same-token frame can reserve or project. Exit publication waits + for the held contiguous prefix to admit or for cancellation proof whose + sent end covers every privately observed exact-token frame. +30. Transfer does not consume rollback authority until the ledger proves the + lease is still current and installs the private recovery sink. A stale + transfer can still retire and cancel only its own token. +31. Canceled-frame suppression retains at most one latest ordered delivery + identity per relay PTY. Transport ordering makes earlier token records + obsolete when a later cancellation is observed; exact next activation, + PTY exit, or provider teardown clears the remaining record. + +## Protocol + +### Architecture decision record + +Decision: `PtyConsumerSession` is a shared semantic state machine, not a +universal transport protocol. The architecture gate is closed and is no +longer waiting for exact wire/API names: the SSH adapter implements +`sourceActivation`, source-bearing `pty.data`, `pty.recoveryComplete`, +cumulative `pty.ackData`, and token-scoped cancellation. Current-working-tree +deterministic validation is an engineering gate; unexecuted live topologies +remain separate promotion gates. A relay-only +`pty.getCapabilities` followed by `pty.negotiateClient` remains rejected +because it would create a second readiness authority beside the existing +authenticated handshakes. + +The 2026-07-28 architecture/terminal review closed its three blocking +findings as follows: + +| Finding | Implemented resolution | +| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Exited delivery removed while uncredited spans remain | Native exit seals the token; logical state survives until cumulative ACK, exact transfer, token-scoped cancellation proof, or generation close. | +| Terminal obligations depended circularly on ACK publication | Obligation terminality advances `obligationsTerminalEndSu`; the coalescer independently queues `ackQueuedEndSu`; only mux write settlement advances `ackPublishedEndSu`. | +| Desktop projection admission lacked immutable range identity and rollback | Model, source span, projection range, and scanner snapshot reserve atomically; pre-commit failure rolls back, while post-commit replacement transfers with proof. | + +Later adversarial review also required exact recovery continuity, stale-attempt +isolation, session-grant fallback semantics, and idempotent remote detach +after token reclamation. Those are incorporated in the activation, lifecycle, +cleanup, tests, and compatibility sections below. + +The final 2026-07-28 architecture/terminal re-review found and closed three +additional interleavings: + +| Finding | Implemented resolution | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Failed exit publication could discard a sealed delivery before its suffix was credited | Failed publication retains the sealed delivery; late cumulative ACK and exact owner recovery can still settle it, and exit is republished without reopening source admission. | +| Recovery-response cancellation authority ended at enqueue rather than writer settlement | The request remains cancellation-authoritative until its response settles; retry reuses the same provisional replacement identity instead of creating an unfenced owner. | +| Snapshot replacement transferred every pending remote span at one sequence fence | Each immutable span retains its model-sequence end; replacement transfers only spans covered by the authoritative `SnapshotEnd`, while trailing spans remain live and replay in order. | + +Current-working-tree adversarial reconciliation closed these additional integrated +failures without widening the approved topology: + +| Finding | Implemented resolution | +| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Invalid-checkpoint `restoreRequired` left a canceled delivery blocking retry | Keep the exact canceled record through response settlement, then delete it only if it is still current; retry mints a fresh activation token. | +| Recovery cancellation publication/proof rejection left ambiguous authority | Raise a typed recovery-cancellation failure and close the exact provider generation, disposing its mux, provider, publishers, lease, and registration exactly once. | +| Remote partial ACK or higher-generation token rotation detached the stream | Release exact cumulative byte credit at any valid interior offset without source settlement; admit only contiguous strictly higher client/owner generations with a new token. | +| Snapshot cleanup raced cancellation-proof reclamation | Make ledger replacement commit idempotent and make remote replacement commit reject reclaimed spans while rollback, settle, and detach prune them without dereference or new credit. | +| Drain resumed the main mux's old ordinary FIFO ahead of control | Select liveness, control, and ordinary from separate FIFO lanes; prioritize control after drain and force one ordinary write after four controls. | +| Same-turn source data escaped a provisional activation | Queue data privately under the provisional lease; commit publishes in order, while rollback drops it, restores only the active predecessor, and awaits exact cancellation proof. | +| Exit cancellation proof reclaimed spans before projection transfer | Acquire proof first, transfer projections, commit proof, prepare/finalize once, and fence every post-proof step against provider-generation close. | +| Exit during private recovery could publish before held frames settled | Seal the private lease immediately, wake the exact recovery fence, then either admit the ordered prefix before exit or retire it only after exact token-cancellation proof. | +| A superseded transfer lost its cancellation authority | Keep the outer lease unsettled until transfer ownership is verified; a stale transfer can still roll back and request cancellation for its exact token. | +| Recovery proof watermark skipped rejected or post-restore frames | Observe every exact-token private frame before retention or restore checks and reject any cancellation proof whose sent end is below that immutable high-water. | +| Canceled-token tombstones grew once per token | Retain only the latest ordered canceled token per relay PTY, replace it on later cancellation, and clear it at exact next-activation, PTY-exit, or provider-teardown boundaries. | +| Reconnect checkpoint raced a running raw model callback | Freeze the exact provider-generation/PTy admission, cancel only queued work, and await the bounded running callback before exporting a checkpoint. | +| Migration-owned callback failure closed the shared provider | Contain failure to the exact migrating PTY, reset only its model, preserve siblings and transport, and retain the ordinary generation-fatal policy outside migration. | +| Closed-generation telemetry omitted active gaps | Compact exact closed ranges and expose the count of allocated but unclosed provider generations below the closed high-water. | + +The final exact-implementation-head architecture/lifecycle review was clean +across 27 files and 296 tests. The transport/topology review confirmed the +exact-PTY containment fix and found no remaining code blocker; its sole +evidence finding was the stale artifact record replaced by the +provenance-bound build and Docker run above. + +The common state machine accepts only: + +- an authenticated principal and owner-eligibility decision from the adapter; +- a consumer generation and optional capability offer/grant; +- a subscription/delivery identity and close reason; +- an adapter-provided publication fence. + +It does not authenticate sockets, parse frames, own sentinel/residue bytes, +wait for stream drain, prove reconnect credentials, or define remote-runtime +encoded-byte ACKs. Those remain adapter responsibilities. The semantic input +and output may be represented as: + +```ts +type PtyConsumerSessionHello = { + clientInstanceId: string + requestedRole: 'session-owner' | 'subscriber' + resume?: { + ownerGeneration: number + ownerLease: string + } + capabilities?: { + outputFlowControl?: { versions: [1]; requestedWindowSu: number } + } +} + +type PtyConsumerSessionGrant = { + serverBuildId: string + clientGeneration: number + role: 'session-owner' | 'subscriber' + ownerGeneration?: number + ownerLease?: string + capabilities?: { + outputFlowControl?: { version: 1; windowSu: number } + } +} +``` + +These types are not wire schemas. Authentication produces a transport-bound +principal and `allowSessionOwner` before the state machine sees the offer. A +request field cannot self-promote a client. The state machine performs +generation allocation, owner replacement, capability intersection, +publication fencing, and close cleanup; the adapter proves identity, carries +the semantic fields, and calls the fence. + +The reviewed decision preserves the existing connection machinery: + +| Path | Authentication and identity | Binding in this PR | +| --------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Local in-process provider | trusted main-process construction | unchanged; it does not negotiate SSH V1 | +| Local daemon | token-authenticated `HelloMessage`, stable `clientId`, paired control/stream sockets, and `daemonIdentity` | unchanged; an exact-version daemon remains outside the SSH wire contract | +| SSH relay socket/named pipe | existing authenticated relay endpoint and dispatcher client identity | first framed `pty.openClient`; response settlement is the readiness fence | +| Primary relay stdio and WSL child stdio | no authenticated dispatcher principal in this PR | bounded legacy transport only; an unproved client cannot become session owner | +| Remote-runtime server | paired-device/E2EE identity and connection ID | source-range reserve/commit/rollback only; never receives an SSH owner lease | + +The implemented owner path is the authenticated detached relay endpoint used +by the SSH provider. After extracting the exact sentinel and residue, main +sends `pty.openClient` before spawn, attach, or ordinary dispatch. The relay +adapter queues later decoded frames behind the session response's write +callback, so an eager next request cannot race activation. Main marks the +provider ready only after receiving the valid response. + +Detached `--connect` first reads the version-scoped endpoint credential and +authenticates its Unix-socket/named-pipe bridge before emitting its sentinel. +The adapter converts that proof into a principal and +`allowSessionOwner = true`; `pty.openClient` carries no secret to the generic +RPC handler. The primary relay stdin/stdout dispatcher identity remains +explicitly unproved and therefore subscriber-only; there is no launch nonce in +this implementation. WSL child stdio shares the bounded decoder/writer +transport changes but does not negotiate an SSH V1 owner in this PR. A manual +relay without the version-scoped endpoint credential is also subscriber-only. + +POSIX hosts may expose different shell and SFTP home namespaces. While the +install lock is held, a per-launch marker maps the canonical shell credential +path to the matching SFTP-relative path; the credential write uses that +mapping before the lock is released. If the marker is unavailable or its +creation fails with confirmed termination, main generates the credential +remotely with the selected Node binary at the canonical shell path. System SSH +keeps its shell-path writer, Windows keeps its native secure writer, and the +remote CLI launcher receives the same credential-file path. An unconfirmed +marker teardown retains the lock rather than authorizing a competing writer. + +The review compared sentinel extension and the single request against one +readiness authority, no first-spawn race, authenticated owner replacement, +response publication fencing, reconnect idempotency, exact residue transfer, +and deterministic testability. The single request wins for the authenticated +deployed-relay bridge. A separate capability probe, two-step negotiation, and +a generic cross-transport wire API are not approved. + +The local daemon adapter is deliberately not changed in this PR. Its existing +sequential control/stream hello, authentication, exact-version fencing, and +prior-version isolation remain authoritative. Reusing the semantic state +machine there is a later design decision and is not required to ship the SSH +bound. + +Every new SSH session offers V1, and every relay from the same build advertises +V1. The server intersects the offer with its supported versions and clamps +`windowSu`; an older client that omits the capability receives bounded legacy +delivery while still establishing the role and generation. Main accepts only +a returned version it offered and a finite safe integer satisfying +`0 < windowSu <= requestedWindowSu`. An absent or invalid V1 grant after +`pty.openClient` succeeds closes that connection attempt rather than silently +downgrading it. + +For compatibility with a legacy relay that does not implement +`pty.openClient`, main permits token-free bounded legacy delivery only for the +narrow same-build-validated JSON-RPC method-not-found result. Other errors, +missing grants after a successful method call, and unproved version skew fail +closed. This fallback does not change the normal same-build contract: the +session offers V1 and the relay advertises and grants it. + +“Bounded legacy” is transport backpressure, not a hidden source-credit mode. +It creates no delivery token, source window, or cumulative ACK obligation, and +the relay continues to ignore legacy `pty.ackData`. Slice 3 retains at most +2 MiB of ordinary publications per client and 32 MiB across the relay, with +1 MiB/24 MiB low waters, plus at most one 128 KiB producer-held frame per PTY. +At a cap, `PtyHandler` pauses the PTY; accepted write callbacks/drain remove +transport publications, and crossing both low waters resumes it. Slice 2/4 +read pauses propagate through SSH until the relay writer saturates, so stalled +model admission remains bounded without pretending renderer ACKs reached the +relay. + +Legacy exit waits only for preceding transport publications and the exit write +callback; main's receive-exit barrier below still waits for its admitted model +and projection work. Transport close cancels the connection's retained +publications and reconnect uses the existing replay/restore behavior. It may +pause all legacy subscribers behind one slow connection because legacy mode +has no authenticated source-owner role, but it neither drops output nor grows +without bound. These mechanics remain available to negotiated legacy peers. + +The session hello creates no PTY token. A later V1 spawn/attach returns a fresh +`deliveryToken` and creates a subscription only for the authenticated +transport client and installed generation. `pty.attach` must receive request +context just as `pty.spawn` does. Spawn failure, identity mismatch, stale +context, and response cancellation create no active V1 subscription. A legacy +spawn/attach returns the current identity/replay shape and creates only the +bounded transport subscription above—no token or source coordinate. A live V1 +token never rotates into legacy service; disconnect closes it before any +separately negotiated legacy fallback connection can open. + +Session ownership is granted by the authenticated session hello, not +constructor-assigned: + +- fresh relay deploy writes one endpoint credential beside the versioned + endpoint, reusing the local daemon's token-authenticated-hello pattern rather + than inventing a second claim protocol. It has mode `0600` on POSIX or a + current-user ACL on Windows. `--connect` proves it; a plain dispatcher socket + without the credential is subscriber-only. Primary relay stdio and WSL + child stdio remain unproved bounded-legacy transports. The adapter converts + proof into `allowSessionOwner = true` and does not expose the credential to + RPC handlers; +- the main creates one opaque `clientInstanceId` for its relay-session + lifetime. The first authenticated owner hello when no owner exists binds + that identity, returns a lease, and starts owner generation 1; +- until that first grant crosses its write fence, a retry with the same + authenticated identity is idempotent and any competing identity is rejected. + Once fenced, replacement requires both the current lease and expected + generation; +- an unexpected owner disconnect retains that lease and its PTY grace cursors + for 30 seconds. A reconnect presenting the lease and expected owner + generation atomically increments the generation, rotates the lease, replaces + the old client, and transfers grace ownership before any new token activates; +- a same-generation retry is idempotent only for the same live client. + Competing, stale, lease-less, or unauthenticated identities cannot replace + the owner and may connect only as ordinary subscribers; +- after grace expiry, the old owner lease is invalidated and the next + owner-authorized session hello can elect a new owner at the daemon's next + monotonic owner generation; generations are never reused. Ordinary + subscribers never inherit owner teardown protection merely by arriving first. + +Source activation, `pty.data`, recovery completion, ACK, and cancellation +carry client/owner generations and the delivery token. `pty.exit` retains its +existing PTY ID plus incarnation shape and resolves against the already +installed delivery identity; it cannot create or rotate one. Main rejects a +generation other than the one installed by its synchronous response hook. The +constructor stdout client is never implicitly a session owner. Production +launches detach on POSIX and Windows, invalidate stdout, and connect the +desktop bridge through `attachClient` over the versioned Unix socket or named +pipe. Unproved direct stdio remains subscriber-only. + +Capability support is negotiated on every SSH session. `SshRelaySession` +always offers V1 during initial connection and automatic reconnect, and POSIX +and Windows relays always advertise it as an unconditional capability. Sink +drain, writer ordering, decoder bounds, and header-ACK hardening remain +unconditional correctness fixes. + +Production deployment does not form arbitrary mixed-build main/relay pairs. +`computeRemoteRelayDir` content-hash-scopes the install directory and its +socket/named-pipe endpoint, and the `.version` handshake is a second fence, so +the desktop bridge and daemon reached at that endpoint share a build. The +normal same-build mode is session-granted V1. A missing mandatory session grant +fails readiness except for the narrow method-not-found compatibility fallback +above. Protocol tolerance for unknown fields remains for direct/manual relay +launches, but an absent session contract outside that fallback is diagnostic +`unsupported-version-skew`, not a deployment cohort. + +The reachable upgrade skew is an orphaned prior-version daemon in its old +version directory with live PTYs while the new main connects to a new endpoint. +It retains its old behavior until its own grace/cleanup completes and is not +reachable by a new same-build session. Upgrade diagnostics enumerate +these versioned orphan processes; V1 neither adopts their PTYs nor claims to +bound their memory. + +Unknown fields are never capability proof. Place the transport-neutral state +types with the narrowest shared session package, while relay/daemon/runtime +wire types remain in their current protocol packages. Do not make the local +provider depend on relay framing for code reuse. + +### Data and ACK schema + +Flow-controlled output is: + +```ts +// relay -> one subscribed client +{ + jsonrpc: '2.0', + method: 'pty.data', + params: { + id: string, + ptyIncarnation: string, + data: string, + deliveryToken: string, + clientGeneration: number, + ownerGeneration?: number, + sourceEndSu: number, + sourceLengthSu: number, + seq?: number, + rawLength?: number, // required and equals sourceLengthSu when transformed + transformed?: true + } +} +``` + +For untransformed frames, `sourceLengthSu = data.length` and `rawLength` is +absent or equal. For `transformed: true`, `rawLength` is required, finite, +safe, non-negative, and equals `sourceLengthSu`; display length is never used +as source credit. A violation cancels the token as malformed. `sourceEndSu` is +the monotonic cumulative coordinate within one opaque `ptyIncarnation`, not a +display offset. A new token starts with `sentEndSu == creditedEndSu` at its +declared checkpoint, so its absolute coordinate still satisfies the window +equation. The source interval is +`[sourceEndSu - sourceLengthSu, sourceEndSu)`. It is independent of `seq`, +which remains the terminal-model source sequence. + +The wire does not send `providerGeneration` or `spanId`. Main supplies its +locally allocated provider generation and derives +`spanId = deliveryToken:sourceStartSu:sourceEndSu`; together with `id`, +client/owner generations, incarnation, token, source/display ranges, and +transform metadata, that forms the immutable admission identity. No later +queue, snapshot, ACK, detach, or replacement path may reconstruct identity +from PTY ID, byte count, or display length alone. + +ACKs are cumulative and coalescible. Main holds the latest settled end per +token and emits at most one batched notification per SSH session every 8 ms, +or immediately when any token frees at least 64 Ki su: + +```ts +{ + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: Array<{ + id: string, + deliveryToken: string, + clientGeneration: number, + ownerGeneration?: number, + creditedEndSu: number + }> + } +} +``` + +One frame contains at most 64 latest-value entries; another turn handles the +remainder. Replacing a queued entry for the same token is lossless because the +value is cumulative. ACK frames use the reserved control lane and never one +frame per data frame. + +The relay normally accepts an entry only from the owning dispatcher client and +only when `creditedEndSu` is a finite safe integer satisfying +`previousCreditedEnd <= creditedEndSu <= sentEnd`: + +```text +if creditedEndSu > previousCreditedEnd: + creditedEnd = creditedEndSu +``` + +Negative, non-finite, unsafe, fractional, over-credit, wrong-client, wrong-PTY, +unknown-token, and stale-token values are rejected and counted diagnostically; +they never clamp into valid credit. Duplicate and regressing values are +no-ops. Token generation, not PTY ID reuse, defines the credit lifetime. + +The writer callback is not guaranteed to run before main processes and ACKs an +accepted frame. If an ACK reaches the relay while that exact same-token send +reservation is pending, the ledger records only its exact source-end boundary; +`sentEndSu` and `creditedEndSu` do not advance yet. Successful send settlement +commits the boundary and immediately applies the reservation. Failed +settlement rolls back the send but retains the reserved boundary, so the next +send must reproduce that exact boundary and cannot admit a shorter or later +slice. Cross-token, interior-boundary, and excessive early ACKs remain errors. + +Explicit cancellation is a request so main receives proof: + +```ts +// main -> relay +{ + method: 'pty.cancelDelivery', + params: { + id: string, + deliveryToken: string, + clientGeneration: number, + ownerGeneration?: number + } +} +// result +{ canceled: true, sentEndSu: number, creditedEndSu: number } +``` + +The relay validates client/PTY/token ownership, changes the token to `closing`, +removes its cursor and activation/exit fences, then responds through the control +lane. Duplicate cancellation of the same recently closed token is idempotent. +Main may discard open obligations only after this response drains back through +the mux or the client-generation close proves equivalent cleanup. + +Every relay-initiated token close emits this metadata-only control +notification before the token record is forgotten: + +```ts +{ + method: 'pty.deliveryCanceled', + params: { + id: string, + deliveryToken: string, + clientGeneration: number, + ownerGeneration?: number, + reason: string, + sentEndSu: number, + creditedEndSu: number, + remainingStartSu: number, + remainingEndSu: number, + replacementDeliveryToken?: string + } +} +``` + +It covers supersession, activation/exit timeout, reconnect-grace expiry, and +explicit delivery cancellation. `remainingStartSu == creditedEndSu` and +`remainingEndSu == sentEndSu` state the exact source interval still unsettled +at the relay. If the sink cannot drain the proof, its generation close is the +proof. Without a replacement, main cancels the matching remaining obligations +and schedules restore/reattach. With a replacement, it keeps those obligations +in `transferring` until replacement `pty.data` and its fence prove exact contiguous +coverage, then atomically transfers the covered suffix. Any prefix already +included in the proved model checkpoint is canceled as superseded without +re-ingestion; an uncovered or mismatched remainder cancels with +`restoreRequired` instead of being credited. Every source subrange therefore +settles once. Stale generations are ignored. + +### Activation, replay, and idempotent spawn + +Token creation and response publication are one fenced operation. The exact +activation metadata carried by V1 spawn and attach responses is: + +```ts +type PtySourceReceivingActivation = Readonly<{ + status: 'pending' + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string + checkpointSourceEndSu: number + recoveryEndSu: number +}> + +type SpawnResult = { + id: string + incarnationId: string + sourceActivation?: PtySourceReceivingActivation +} + +type AttachResult = { + incarnationId: string + replay?: string + sourceActivation?: PtySourceReceivingActivation + sourceRecovery?: + | PtySourceReceivingActivation + | Readonly<{ status: 'restoreRequired'; reason: string }> +} +``` + +The dispatcher places the metadata-only response on the response lane and +invokes the request's `onResponseSettled` callbacks only from the exact writer +settlement. Success clears relay activation; failure cancels a fresh token or +leaves the exact recovery attempt available to its proved replacement. +Enqueue, `write(true)`, and `write(false)` alone are not activation proof. + +The main mux supplies the receive fence. Spawn and attach requests register a +synchronous `beforeResolve(result)` hook in their pending-request record. +`handleResponse` parses the whole response and provisionally installs +`sourceActivation` before resolving the Promise. The lease captures the exact +prior receive state, candidate token, and a private ordered list of same-turn +source frames. Spawn and non-recovery attach commit only after operation-ID +result, incarnation, claimed-owner, attach identity, stale-attempt, and exit +validation. Reconnect first validates the owner, incarnation, token, +checkpoint, and recovery end, installs the exact private quarantine, then +transfers the lease to its private sink before awaiting the body/completion +fence. That transfer cannot update ordinary provider listeners or +`livePtyIds`, and it cannot reach the model, runtime, desktop, or remote +consumers. Final commit switches future exact-token frames to ordinary +publication only after the body, fence, model admission, and activation +validation succeed. Any failure before transfer +retires the candidate and drops its held data, restores only the nearest +non-retired predecessor, then waits for a valid `pty.cancelDelivery` response +from the originating mux. After transfer, recovery cancellation or +provider-generation close owns cleanup; rollback is no longer authoritative. +It cannot erase a newer revision or cancel a newer token. An identical +already-committed activation is an idempotent no-op; an identical provisional +activation is stale. Fresh-spawn rollback that cannot prove cancellation +surfaces `execution_owner_unavailable` rather than claiming the old token +retired. + +The receive cursor starts at `checkpointSourceEndSu`, not +`recoveryEndSu`. Source-bearing notifications for the installed identity may +then advance only contiguously. Frames through `recoveryEndSu` stay +quarantined as recovery; later frames stay held until the exact completion +fence commits. Unknown or mismatched source identities are rejected before +`livePtyIds`, runtime, renderer, or source obligations change. Exact +source-bearing frames carry their immutable `ptyIncarnation` directly and do +not invoke or mutate the legacy incarnation resolver. + +Every exact-token private frame advances a monotonic observed-source high-water +before quarantine retention or `restoreRequired` checks. If exact exit arrives, +main wakes the recovery fence and seals the private lease immediately. It may +publish exit only after the held contiguous prefix is admitted, or after +`pty.cancelDelivery` proves `sentEndSu` covers that high-water and the accepted +checkpoint remains the credited end. Capacity-rejected and post-restore frames +therefore cannot disappear below the cancellation proof. + +Output produced while a token is activating is retained by its bounded shared +cursor and can pause the native PTY; it is never sent early. Output arriving +after main's synchronous install but before authority commit is bounded by the +provisional lease and is not visible to runtime, desktop, remote consumers, or +`livePtyIds`. Main drops unknown-token notifications before any side effect. + +Before issuing attach or any same-client replacement, main serializes each +`(providerGeneration, appPtyId)` through a migration fence: freeze new +old-generation intake, cancel queued-but-unstarted model entries, await the one +in-flight emulator callback, then record the last completed receipt. A later +overlapping reconnect preserves and awaits an earlier fence for the same PTY; +an empty newer generation cannot expose the earlier checkpoint. Before the +deadline, every old callback remains owned by its admission and generation +until raw completion settles. Stale-owner retry invalidates the checkpoint +value but retains the same fence, then requests restore after it settles. If +the 10-second migration deadline expires or the callback fails, admission +checks the exact active migration before its generation-fatal default. Main +cancels only that admission, detaches that PTY's old headless model, prefers +provider/renderer snapshot restore, and requests `checkpointUnavailable`; +recovery reports `restoreRequired` instead of replaying a guessed gap. The +shared mux, sibling PTYs, filesystem, and Git traffic remain live. Outside an +active exact-PTY migration, the same callback failure still closes its provider +generation. Late raw completion stays on the detached model and cannot advance +the canceled source checkpoint. + +Attach sends `sourceRecovery: { status: 'checkpoint', clientGeneration, +ownerGeneration, ptyIncarnation, deliveryToken, acceptedSourceEndSu }`. A +queued, in-flight, or merely rendered span is not a checkpoint. Without a +proved fence it sends `{ status: 'checkpointUnavailable' }`. + +The relay accepts the checkpoint only for the same incarnation and within +`[oldCreditedEndSu, retainedLiveEndSu]`. It never clamps a value forward, +trusts a value beyond sent/retained data, or replays freed pre-credit source; +an invalid or uncovered checkpoint cancels the attempted delivery and returns +the token-free `restoreRequired` arm. The canceled record remains installed and +cancellation-authoritative until that metadata response settles. Settlement +deletes it only if the delivery map still points to the exact record and wakes +capacity; a retry can then mint a fresh token. It does not activate a +subscription or stream a partial gap; main either restores an authoritative +model generation before retry or surfaces the gap. + +The reconnect coordinator consumes `restoreRequired` and owns snapshot-backed +replacement. A generic existing-session provider reattach has no such recovery +owner, so it maps `restoreRequired` to `SSH_SESSION_EXPIRED` instead of +returning an outputless live PTY; the existing renderer fallback may then +replace that stale session. + +Recovery bodies use the ordinary source-bearing `pty.data` schema. The new +token initializes `sentEndSu = creditedEndSu = checkpointSourceEndSu`, and +the retained gap covers exactly +`[checkpointSourceEndSu, recoveryEndSu)`. Each frame consumes the normal +window and bounded model admission; there is no separate recovery body method, +stream ID, or replacement-only snapshot kind on this SSH wire. + +Only after the last recovery `pty.data` writer callback succeeds does the relay +attempt this metadata-only control notification: + +```ts +{ + method: 'pty.recoveryComplete', + params: { + id: string, + clientGeneration: number, + ownerGeneration: number, + ptyIncarnation: string, + deliveryToken: string, + checkpointSourceEndSu: number, + recoveryEndSu: number + } +} +``` + +`recoveryCompletionPending` permits one admitted attempt at a time. If control +capacity rejects it before admission, the record retains both range ends and +the capacity callback retries; repeated capacity signals cannot enqueue +duplicates. Only the completion notification's successful writer settlement +clears recovery state and permits buffered live `pty.data`. A failed +settlement leaves recovery fenced for exact owner replacement or cleanup. + +Notification-style `pty.replay` remains the legacy attach compatibility path. +`pty.serialize` remains its existing bounded metadata response; neither is a +second V1 source-recovery body channel. + +Main quarantines the entire candidate recovery transaction until it can prove +the fence. The first source-bearing recovery range must start at the accepted +checkpoint, every later range must be exactly contiguous, and the final end +must equal `recoveryEndSu`. Empty recovery is valid only when the checkpoint +already equals `recoveryEndSu`; main retains that end as the required start of +the first later live frame even after activation. The first live frame and +every successor must also be exactly contiguous. Gap, overlap, incomplete +suffix, missing body, bad empty recovery, or bad live handoff reaches neither +the model nor desktop projection. + +A failed or stale recovery attempt cancels only its own replacement token. +It never sends cancellation on a newer mux, clears a newer checkpoint, shuts +down the physical PTY/process, deletes PTY ownership, or expires the owner +lease. A valid matching token-scoped cancellation proof is applied locally and +leaves the lease detached and retryable. If the current attempt cannot publish +cancellation or local proof application rejects, the session fails closed: +it closes the exact provider generation with +`ssh_source_recovery_cancellation_failed`, disposes that generation's mux, +provider, ACK/cancellation publishers, provisional lease, and registration +once when still provisional, or its transferred activation state after +quarantine ownership begins, and reconnects without clearing physical PTY +ownership. If the attempt is already stale, it performs no cleanup against the +replacement. A fresh +relay that no longer retains the cached owner returns typed error `-32041`; +main clears only the cached owner/checkpoints and retries `pty.openClient` +exactly once without `resume`. + +Idempotent agent-session spawn has two layers. The cached +`agentSessionCreateOperationId` promise returns only the physical PTY +identity/outcome. After that promise resolves, every current, non-stale outer +`pty.spawn` request creates its own requesting-client subscription and +activation fence. Creating a token for an existing +`(clientId, clientGeneration, ptyIncarnation)` is one atomic supersession +transaction: create the replacement cursor first, move the old token to +`closing`, cancel its outstanding activation/exit work, emit +`pty.deliveryCanceled(reason='superseded')`, and return the old token in the +new response. The cancellation names the replacement token and exact remaining +old span. Main transfers that span only after matching recovery completes; +otherwise it cancels it and restores. The pair therefore has at most one +active token and no range is both canceled and replayed as new model input. A +retry served from the cache receives one fresh token, never a duplicate live +subscription. If the client becomes stale after physical commit, no token +survives; the next retry can subscribe to the retained PTY. + +## Relay output architecture + +Split policy from lifecycle and transport: + +- `pty-source-credit-ledger.ts`, `pty-source-credit-record.ts`, and + `pty-source-replay-index.ts` own cumulative token state, immutable retained + source records, exact replay ranges, rotation, and reclamation. +- `relay-pty-source-publication.ts`, + `relay-pty-source-activation.ts`, + `relay-pty-source-send-scheduler.ts`, and + `relay-pty-source-exit-publication.ts` own activation/restore settlement, + 16 Ki-su publication, recovery, exit barriers, budgets, and subscriber + scheduling. +- `dispatcher-client-writer.ts` owns the one writer per sink, activation + fences, lane queues, drain callbacks, and close settlement. + +`PtyHandler` retains PTY lifecycle, the 8 ms initial/1 ms continuation cadence, +immediate interactive/transformed publication, replay tail, streaming +transform state, and idempotent native `pause()`/`resume()`. It publishes +immutable source spans rather than broadcasting notifications. + +### Cumulative span ledger + +Each PTY ledger is append-only until every cursor has passed a span: + +```ts +type RelaySourceSpan = { + spanId: number + sourceStartSu: number + sourceEndSu: number + data: string + splittable: boolean + retainedBytes: number +} + +type PtyDelivery = { + state: 'activating' | 'recovering' | 'active' | 'sealed-unsettled' | 'closing' | 'closed' + clientId: number + clientGeneration: number + ownerGeneration?: number + deliveryToken: string + cursor: { spanId: number; displayOffset: number; sourceOffsetSu: number } + sentEndSu: number + creditedEndSu: number +} +``` + +Different subscribers may stop at different offsets in one span. A cursor +contains both display and source offsets, so no global chunk split can lose or +repeat a suffix. Data storage may coalesce adjacent splittable spans for a +frame or slice one span per cursor; the immutable source boundaries and +cumulative ends do not change. Reclamation requires every live, activating, +sealed-unsettled, closing, or reconnect-grace cursor to pass the complete +span. Native process exit seals admission but does not weaken reclamation. + +The transform publisher cuts source input into scalar-safe pieces, never +between a surrogate pair, and records each display result against its source +interval. An untransformed span may be 16 Ki su and is sliced per sink. +A transformed span is indivisible after publication, so its publisher reduces +the source slice until the encoded span is at most 8 KiB, below the token +window, and no larger than the sink's empty non-reserved capacity. An +implementation that cannot stream a transform must pause before the hard limit +and request model restore; it may not over-credit, truncate, or send an +oversized or permanently inadmissible frame. + +The existing 100 Ki-su replay-tail policy and the chunked +`RecentPtyOutputBuffer` representation from `79ec57d04` remain. +`PtySourceReplayIndex` now provides the bounded parallel source-range index +without restoring the former rolling-string re-slice. Legacy replay still +materializes the same display tail. V1 restore additionally preserves +incarnation, exact source ranges, and transform metadata. Index eviction +follows the buffer's exact retained prefix and never fabricates a mapping; the +existing append-path equivalence/performance contracts remain in force. + +### Scheduling and fairness + +The relay scheduler uses deficit round-robin over PTYs, then rotates subscribed +clients for the selected PTY. One scheduling turn admits at most: + +- two PTY frames; +- 32 Ki su; +- 2 ms of scheduler work. + +Requeue work with `setImmediate` after any limit. Input and control requests can +therefore interleave with output. A recent-input PTY may move to the front once +per round, but repeated interactive classification cannot consume another +PTY's quantum. Preserve the existing bounded interactive fast path: after any +already-queued liveness/control frame, if its token is active, has no earlier +backlog, remaining window covers the exact slice, and the writer admits it +without reserve use, publish one echo frame of at most 4 Ki su immediately +instead of waiting for the 8 ms timer or DRR. Allow at most one such frame per +input epoch; it advances the same cursor once and then yields to normal +priority selection. + +For each live/interactive candidate, all gates must be open: + +```text +frameSourceLengthSu <= windowSu - (sentEndSu - creditedEndSu) +client writer admits PTY bytes without consuming its control reserve +token state == active +``` + +For a splittable span, the admitted slice is at most +`min(16 Ki su, remainingWindowSu)`. An indivisible transformed span is admitted +only when its entire source length fits. There is no one-frame window +overshoot. Dispatcher admission creates one pending send reservation. A +`write(false)` still gives Node that frame exactly once, but the cursor and +`sentEndSu` commit only on successful writer settlement; failure rolls back +the reservation. No later ordinary frame is admitted meanwhile. Unsent frames +remain ledger-owned. Liveness, control, interactive PTY, ordinary source PTY, +and bulk are reconsidered on every writer turn. Recovery and live output use +the same source-bearing `pty.data` producer; recovery additionally requires +`token state == recovering`, exact continuity, byte capacity, and +remaining-window capacity. + +The V1 source owner and additional subscriber projections use independent +publication transactions: +`projectPtyDataToMatchingClients`/`projectPtyExitToMatchingClients` exclude +the `source-owner` and reserve each remaining client separately. If one +additional subscriber cannot reserve bounded writer/publication capacity, +close only that client and continue the healthy subscriber and source-owner +sends; never pause or tear down the native PTY solely for that projection. A +legacy-fallback primary is different: it remains a required bounded-legacy +backpressure participant and is not evicted for saturation. Constructor +position still grants no session-owner role. + +Pause when every remaining delivery is blocked or either retained-byte hard +cap is reached. Resume only when: + +```text +perPtyRetainedBytes <= 1 MiB +relayRetainedBytes <= 48 MiB +at least one delivery can advance, or no delivery remains +``` + +Both predicates are required; the per-PTY low water cannot immediately resume +while the relay-wide budget remains exceeded. Calls are idempotent because a +last `node-pty` callback may re-enter pause/exit handling. + +### Exit + +On native exit, seal the PTY's output stream after publishing the last ingress +emissions. The source-owner delivery changes to `sealed-unsettled`; additional +subscriber exits use their independent bounded projection transactions: + +1. write all preceding data within normal window and drain rules; +2. once that data is accepted by the sink, write `pty.exit` without waiting for + the final data ACK; +3. record the exit write callback as `exitPublished` only on explicit success; + on error, generation-close/cancellation proof owns cleanup. Retain the + delivery, token ledger, cursor, cumulative ACK state, and timeout while any + `[creditedEndSu, sentEndSu)` suffix remains; +4. close only when the outstanding suffix becomes terminal through cumulative + ACK application at the relay, exact transfer to a replacement, explicit + cancellation, or client-generation close proof. + +Never force a final tail past the credit window. An additional subscriber that +cannot admit its projected `pty.exit` is detached without changing the source +owner. For the session owner, cancel only this PTY token after its bounded +deadline, keep the sealed record until ACK, transfer, cancellation, or +generation-close proof, and do not tear down unrelated PTYs. A late valid ACK +against a sealed token remains valid and may complete cleanup. Sealed records +are excluded from the 50-live-native-PTY spawn admission count but remain +charged to retained-data budgets. Relay disposal generation-closes all +records and clears their scheduled work exactly once. Exit listeners and +native PTY disposal run at physical exit; logical delivery cleanup waits for +proof. + +Main has a matching receive-exit barrier. Receiving `pty.exit` seals the token +against later data but does not immediately call `runtime.onPtyExit`, retire +the headless emulator, clear desktop ranges, close remote subscriptions, or +send renderer exit. It retains every preceding model receipt, projection ID, +remote mapping, and ACK publication record. In order, it: + +1. waits for preceding model receipts; +2. publishes or exactly transfers committed desktop ranges and required + remote mappings; +3. advances/queues the terminal cumulative source ACK; +4. then invokes runtime and renderer exit cleanup while retaining ACK + publication state until success or generation-close proof. + +The barrier has the same charged 30-second deadline. At expiry, main first +sends token-scoped `pty.cancelDelivery` and waits up to 10 seconds for its +proof without applying it. If the exact barrier/generation still owns the +exit, main transfers published desktop projections to +`ssh-exit-delivery-canceled`, commits the proof to reclaim source spans, +prepares exit once, rechecks generation ownership, finalizes exit, rechecks +again, then closes the projection. This order prevents proof commit from +reclaiming a span before its published projection transfers. + +If provider-generation close wins while proof is pending, it rejects and +removes the barrier plus its preparation ownership; cancellation ownership is +operation-local rather than retained in a process-lifetime tombstone. The late +proof performs no transfer, commit, preparation, final exit, or projection +close. If cancellation publication or proof fails while the barrier remains +current, main closes the provider so generation cleanup proves cancellation; +unrelated PTYs are not torn down on the ordinary successful timeout path. A +late emulator callback cannot mutate the reset generation. Thus relay sink +ordering and main asynchronous model ordering are both proved. + +Provider generations come from one process-wide monotonic allocator and are +never reused. Main rejects closed generations through sorted, merged inclusive +ranges rather than one retained entry per reconnect. Sequential closes compact +to one range; out-of-order closes preserve exact live-generation gaps until +those generations close, then merge. This keeps stale-generation rejection +exact while retained range count follows live gaps instead of process uptime. + +## Dispatcher and transport drain + +Every dispatcher client is constructed with one bidirectional transport: + +```ts +type SinkWriteResult = 'accepted' | 'saturated' | 'closed' +type SinkWriteSettlement = { ok: true } | { ok: false; error: Error } + +type RelayClientTransport = { + write(data: Buffer, onSettled: (result: SinkWriteSettlement) => void): SinkWriteResult + writableLength(): number + writableHighWaterMark(): number + onDrain(cb: () => void): () => void + pauseReads(): void + resumeReads(): void + close(): void +} +``` + +Adapters map Node `true` to accepted, `false` to saturated, known-dead to +closed, and map the `stream.write(data, callback)` callback's optional error +to explicit success/failure settlement. Only `{ ok: true }` is a publication +fence. A callback error or thrown error closes the client, cancels the +generation, and cannot activate a token, publish ACK state, or release a source +obligation. A saturated or closed transport is never called again for ordinary +traffic before drain/replacement. +One 13-byte keepalive may bypass a saturated epoch; no second liveness bypass +is allowed until that write callback or drain, so the exemption is constant +space rather than an unbounded queue. + +`DispatcherClientWriter` is the only encoder/writer for a client. It owns: + +1. a coalesced liveness lane, at most two 13-byte frames; +2. a FIFO control lane, at most 256 frames and 1 MiB encoded; +3. producer-scheduled interactive PTY, ordinary source PTY, and bulk lanes, + whose frames remain with their producers until the writer admits them. + +It reserves +`min(64 KiB, max(1 KiB, floor(highWaterMark / 4)))` below the stream's +high-water mark for liveness/control. PTY, recovery, and bulk admission all +require +`writableLength + frameBytes <= highWaterMark - effectiveReserve`; the +scheduler reduces source slices and reshapable bulk before publish. The +`fs.streamChunk` wire contract is the exception: its fixed 256 KiB raw chunk +and sequence-offset math are preserved, and the `fixed-bulk` lane admits one +encoded frame only when the producer queue and writable sink are empty and +unsaturated. It then obeys ordinary callback/drain settlement before another +producer frame. No producer writes around these gates. +Every transport accepted for a PTY subscription must expose at least 8 KiB of +empty non-reserved capacity. A lower-capacity subscriber is rejected before +token creation, so an already-published indivisible transformed span cannot +become permanently inadmissible. +Liveness is selected first, then control FIFO, interactive PTY, ordinary source +PTY, and bulk; after one producer frame the writer re-runs selection. The +physical stream is still FIFO—V1 does not claim a second SSH channel—but the +finite burst and byte reserve bound head-of-line delay. Control overflow closes +an additional subscriber; overflow on the negotiated session owner is a +transport failure and reconnects that owner generation. + +Drain registration is one-shot per saturated epoch. `close`, `error`, +`attachClient`/`detachClient`, owner-generation replacement, and dispose +cancel every outstanding write callback and producer fence exactly once. +`setWrite` remains a test/compatibility seam; production reconnect ownership is +the socket/stdin construction plus `attachClient` and `detachClient`. + +In detached relay startup, immediately invalidate the synthetic stdout client +when `stdoutAlive` becomes false. A no-op writer must not look like successful +delivery. The first valid session-owner grant on an `attachClient` +socket/named-pipe client elects the owner; reconnect resume replaces its +generation atomically. Constructor identity and connection order confer no +role. + +One `DrainAwareStdoutWriter` owns every byte written to `process.stdout`, +including the initial relay sentinel, handshake residue, dispatcher frames, +and connect-mode forwarding. `runConnectMode` performs +`sentinel -> residue -> socket data` through that writer and pauses the source +socket after stdout saturation; it never mixes ad-hoc `stdout.write()` with +`sock.pipe(stdout)`. The same state contract wraps Unix sockets, Windows named +pipes, and the initial stdio client. + +Keepalive frames are independent of PTY credit, use reserved capacity, and have +the single-frame saturated-epoch exemption above. A decoder self-pause or +local writer-saturation epoch suspends dead-link evaluation. Resume calls the +same `rebaseHealthClocks(now)` used after a wake gap: set `lastReceivedAt` and +every existing `unackedTimestamps` entry to `now`, then allow a full timeout +window. Rebasing only received-data age is insufficient because the +outstanding-header-ACK conjunct would remain stale. During a suppressed +interval, keepalive intent coalesces behind one outstanding probe and the +header-ACK timestamp map has a hard entry cap; at 4095 entries it stops +ordinary main-to-relay admission and reserves the final coalesced entry for +cancel/liveness. +No pause fabricates an ACK. Tests sustain both states past 20 seconds and prove +zero reconnect oscillation or timestamp growth. + +## Main-process credit ownership + +`SshRelaySession.wireUpPtyEvents` remains the provider listener but stops +calling `runtime.onPtyData` and `webContents.send` itself. It validates the +provider generation/token and hands each notification once to a main-only +`SshPtyOutputDelivery` intake installed by `src/main/ipc/pty.ts`. This intake is +the only SSH owner allowed to ingest runtime output, mutate delivery state, or +send `pty:data`. + +### Main cumulative ledger + +The intake appends immutable wire spans: + +```ts +type PtySourceDeliveryIdentity = Readonly<{ + id: string + providerGeneration: number + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string +}> + +type PtySourceSpan = PtySourceDeliveryIdentity & + Readonly<{ + spanId: string + sourceStartSu: number + sourceEndSu: number + displayStart: number + displayEnd: number + data: string + splittable?: boolean + indivisible?: boolean + transform: { + transformed: boolean + rawLengthSu: number + scalarSafe: boolean + } + }> + +type SourceSpanRecord = { + span: PtySourceSpan + obligations: Map +} + +type SpanObligation = + | { state: 'open' } + | { state: 'transferring'; to: ConsumerId; reason: string } + | { state: 'settled'; reason: string } + | { state: 'transferred'; to: ConsumerId; reason: string } + | { state: 'canceled'; reason: string } + +type TokenAckPublication = { + obligationsTerminalEndSu: number + ackQueuedEndSu: number + ackPublishedEndSu: number +} + +type DesktopProjectionSpan = Readonly< + Omit & { + splittable: boolean + projectionSemanticsId: string + } +> +``` + +The ledger is separate from `PendingPtyData`. Queue merge, split, remainder, +drop sentinel, query salvage, thinning, and interactive bypass receive no +delivery token and cannot rewrite source spans. At SSH intake, +`SshPtyLegacyProjectionLedger` keeps per-PTY cursors and admits the complete +immutable `DesktopProjectionSpan`; it never receives only `data` plus source +length. +`PendingPtyData` continues to store display batching fields plus an opaque +projection admission ID, not mutable source accounting. + +`projectionSemanticsId` addresses an immutable per-admission record containing +the accepted chunk's ordered main-authoritative terminal facts and the +projection queue's before/after bounded scanner snapshots. This preserves +current main's dropped-output DEC mode 2031 subscribe/withdraw salvage without +putting mutable scanner state in the source ledger. A source gap resets the +recorded cross-chunk state before later bytes are admitted. + +When the data queue sends a display prefix, drops a pending entry, or replaces +it with salvage, it asks the range queue to consume the same operation. The +range queue—not mutable `rawLength` fields—returns the exact source length and +transform metadata for the renderer payload and consumer obligation. +Admission is transactional: + +1. reserve model bytes and create the source span; +2. stage the main terminal facts and before/after projection scanner snapshots; +3. reserve the exact desktop range by `spanId` and `projectionSemanticsId`; +4. enqueue display data and publish only after every reservation succeeds; +5. commit model ownership, main facts, and projection admission together. + +If model reservation fails, remove every uncommitted reservation. If +projection admission or `webContents.send` fails, roll back the uncommitted +range selection and restores the prior projection scanner snapshot without +publishing staged facts. After model commit, main facts remain published once +and a send failure atomically transfers the desktop obligation plus committed +projection state to a model-restore marker. Queue merge/split preserves the +ordered admission IDs, salvage/drop transfers exact superseded ranges and DEC +mode 2031 decisions, renderer reload transfers committed desktop obligations +before clearing, and token replacement rejects stale IDs by generation while +exact replacement coverage transfers their ranges. Untransformed source may +split only at a recorded display/source offset; a transformed range stays +indivisible. No rollback fabricates an ACK or destroys a live relay +obligation. Renderer ACK retirement reserves both range and ledger mutations, +validates both, then performs one no-throw commit; neither side becomes +terminal before that commit. + +Development assertions enforce: + +```text +receivedSu = openSu + transferringSu + settledSu + transferredSu + canceledSu +obligationsTerminalEndSu = + largest contiguous end whose required obligations are terminal +ackPublishedEndSu <= ackQueuedEndSu <= obligationsTerminalEndSu +``` + +Only `SshPtyOutputDelivery` may call the V1 cumulative ACK coalescer. The +generic `IPtyProvider.acknowledgeDataEvent(id, delta)` API remains for local +and daemon behavior. Legacy-fallback SSH may still emit its current delta, but +the relay ignores it and no legacy bound depends on it; it is a hard no-op for +negotiated V1 PTYs. At every shared `pty:ackData`, resync, heal, +write-off, drop, salvage, and reload call site, V1 routes renderer display +progress through +`SshPtyLegacyProjectionLedger`: parsing settles exact mapped source ranges and +heal/write-off atomically transfers them. These projection transitions may +advance ledger eligibility already earned by the model, but never emit legacy +`{ id, charCount }` wire traffic or manufacture source progress from a display +count. + +ACK eligibility and ACK publication are separate transitions. Terminal +obligations advance `obligationsTerminalEndSu` without waiting for a write +callback. The coalescer independently queues the latest cumulative eligible +end and advances `ackQueuedEndSu`. The mux write callback advances +`ackPublishedEndSu` and permits cleanup only on explicit success; it never +creates eligibility. A synchronous throw, callback error, or close leaves the +cumulative value queued for a generation-aware retry or reaches cancellation +proof on provider close. Coalescing replaces only with a greater cumulative +value, so retries cannot lose an eligible prefix. + +### Bounded asynchronous model admission + +The charged FIFO per PTY and global scheduler replace the unmetered +headless-emulator `writeChain` contract. `acceptPtyData` returns a Promise +receipt: + +1. before capturing the frame, reserve its charged retained bytes against the + per-PTY and global model-admission budgets; +2. the per-PTY budget covers one full token window and the global budget + matches the relay retained-data cap, so a conforming owner reaches token + backpressure before routine admission denial. If a transient frame still + cannot fit, keep that current decoded frame in a separately charged intake + slot and leave its model obligation open. Enter selective pressure mode: + quarantine at most 1 MiB or 64 later PTY data frames in wire order while + continuing to apply transport liveness, unrelated RPC control, and + cancellation proofs. Keep `pty.exit`, `pty.recoveryComplete`, activation + fences, and other source-ordering lifecycle frames behind all preceding + quarantined data for their token. A same-token cancellation proof may bypass + only by atomically canceling those quarantined obligations first. Do not + admit quarantined data out of order. If the reserve fills before capacity + returns, pause reads and start a 10-second provider-close deadline; +3. enqueue only after capacity is owned. Snapshot all consumer memberships and + allow bounded desktop/remote fan-out at that point; neither projection + progress nor queue ownership settles the required model obligation. + Preserve per-PTY order while allowing fair turns across PTYs; +4. resolve the receipt only from the emulator write callback. Then release the + queue charge, settle the model obligation, and advance eligible credit; +5. resume admission only after both low waters hold. Rejection cancels the + token and schedules restore; it never passes through the current + best-effort swallowed-error path. + +The relay writer rechecks control priority after every PTY frame, so the +receive reserve needs to cross only a bounded already-written PTY burst, not an +unbounded producer stream. If no control can be reached within the reserve and +deadline, provider close supplies generation cleanup proof. Selective pressure +never settles, drops, or reorders a data frame and never lets a model stall +silence cancellation indefinitely. + +Provider close, token supersession, PTY exit, and runtime disposal cancel +queued-but-unstarted entries and reject their receipts exactly once. An +in-flight emulator callback owns its entry until completion or failure and +must pass its captured token/generation check before committing. Reconnect and +supersession use the migration fence above rather than letting that callback +race a replacement checkpoint. Failure dispatch consults the exact +`(providerGeneration, ptyId)` migration owner before invoking generation close; +this does not suppress the ordinary non-migrating failure policy. The old +Promise chain may remain as the per-PTY execution primitive only after each +link is charged by this scheduler; it is no longer an unbounded owner. + +### Desktop, mobile, web, and agent policy + +Consumer membership is snapshotted when each span arrives: + +- The main terminal model is always required. + `runtime.acceptPtyDataBounded` returns the asynchronous receipt Promise + above; it resolves only after the emulator accepts the span. + Status and agent-session observers derived from that model add no duplicate + obligation. +- The desktop renderer is a recoverable projection. Parsing settles its + obligation. Hidden thinning, reload, destroyed-window, send failure, + pending-cap replacement, or delivery heal atomically transfers the + obligation to the already-accepted main model and emits a model-restore + marker before settlement. A transfer requested before the model receipt + remains pending and commits only when that receipt succeeds. Once the model + receipt exists, desktop obligations + remain tracked for main-to-renderer bounds but are not required for upstream + ACK eligibility. +- A mobile/web/raw agent terminal subscriber that negotiated lossless ACKs is + required while attached and settles independently from its bounded delivery + cursor. At its ACK cap it stops sending and remains required; a stall cannot + auto-transfer itself out of upstream backpressure. On explicit detach or + replacement, transfer to a snapshot/resubscribe marker if supported; + otherwise cancel that consumer and close only its stream. +- A legacy or observational remote subscriber is best-effort and creates no + upstream obligation. It receives bounded fan-out and is dropped/resynced on + overflow. + +Current mobile notification replay remains a separate runtime method and +identity space. Its notification epoch and post-local-delivery watermark prove +notification catch-up only; terminal obligations require the stream generation +and immutable encoded-byte/source-range mapping below. + +The token ACK advances when the main model and every currently required +lossless consumer for the contiguous prefix have terminal obligations. A +desktop stall therefore cannot freeze a healthy mobile/web consumer, while an +actually lossless remote subscriber still participates explicitly. + +Each lossless remote stream owns an immutable encoded-byte/source-range ledger: + +```ts +type LosslessRemoteSendRange = { + streamGeneration: string + encodedStartByte: number + encodedEndByte: number + providerGeneration: number + ptyIncarnation: string + deliveryToken: string + sourceRanges: ReadonlyArray<{ + spanId: string + sourceStartSu: number + sourceEndSu: number + indivisible: boolean + }> +} +``` + +Existing `terminal.multiplex` `ackOutput: 1` remains a byte-delta flow-control +mechanism and creates no SSH source obligation. A required source-mapped stream +must separately negotiate `ackOutputSourceRanges: 1`. The server allocates an +opaque `streamGeneration` in the subscribed control frame; every cumulative +ACK carries that generation and `ackedEndByte`. A reused `streamId`, old +client, legacy `{bytes}` ACK, or stale generation cannot settle the new stream. + +The mapping is recorded only when the remote writer accepts that exact encoded +output payload, in the same byte unit used by its ACK window. Batching flushes +on provider generation, PTY incarnation, or delivery-token change. When one +encoded frame contains multiple same-identity source spans, it records their +ordered composite list rather than collapsing them into a proportional range. +An ACK must be a finite safe integer in +`previousAckedEndByte <= ackedEndByte <= lastAcceptedByte`; excessive values +are rejected and counted, never clamped into full settlement. An in-range +partial-frame ACK advances the byte watermark and releases exactly that byte +budget, but retains the covering frame's immutable source mapping and settles +no source units. Only complete recorded frame boundaries settle their exact +ordered source ranges. UTF-8 width, JSON escaping, `rawLength`, and display +transforms are never converted proportionally. + +The same attached stream may cross a source-token rotation without transferring +its older unsettled frame. Admission requires the same PTY ID, provider +generation, and incarnation; strictly greater client and owner generations; a +different delivery token; and exact source/display continuity. The ledger then +rebinds future frames to the new identity while older frames retain their own +immutable identity and settle at their encoded boundaries. Equal, lower, +partially advanced, cross-provider, cross-incarnation, or same-token rotations +are rejected. Before stream replacement, snapshot recovery, or detach, create +the replacement owner and atomically transfer every remaining mapping; +otherwise cancel that consumer and its bounded stream. Old stream generations +cannot settle new mappings. + +The main registry stores the immutable source identity with each admitted +remote mapping, not only its `spanId`. A replacement reservation moves the +named consumer obligations to `transferring`; only an accepted headed or +headless `SnapshotEnd` publication at or beyond the reservation's required +sequence commits them. Stale generation, failed publication, disconnect, or +explicit rollback restores the original live-stream obligation. If +token-scoped cancellation proof has already reclaimed the authoritative span, +late ACK/settle prunes the cached mapping, detach skips absent spans, and a +replacement commit returns false without dereferencing the reclaimed record. +Rollback then deletes the exact reservation and skips already-absent spans. +These paths are idempotent and create no credit. + +ACK-pending output overflow uses the same transaction. For a source-mapped +stream, serialize the bounded snapshot, reserve only immutable spans whose +model-sequence end is covered by its `seq` through +`reserveRemoteTerminalSourceRangeReplacement`, publish every snapshot frame, +then call `commitRemoteTerminalSourceRangeReplacement` against the exact +current stream generation, snapshot source identity, and sequence. Only after +commit may the server trim queued chunks with +`chunk.seq <= snapshot.seq` and clear the overflow marker. Serialization, +publication, or commit failure calls +`rollbackRemoteTerminalSourceRangeReplacement`, preserves the live +obligations, reports the stream error, and detaches that stream. One +`ackRecoverySnapshotInFlight` flag bounds the operation to single-flight. +At the encoded-byte ledger, snapshot replacement commit is itself idempotent: +duplicate commits and commits after close are no-ops, preserve cumulative byte +coordinates, and remove only source mappings covered by the authoritative +snapshot sequence. + +The stall policy is deliberately split. Desktop-only parse failure reaches its +bounded projection cap, transfers to model restore, and lets upstream credit +continue. A stalled model receipt or attached lossless remote has no automatic +projection transfer: its required obligation stays open, the token window +exhausts, and the relay pauses the native PTY when every remaining delivery is +blocked. Only an explicit, capability-proven snapshot/resubscribe transition +can transfer a lossless remote obligation. + +### Exactly-once lifecycle + +`settle`, `beginTransfer`, and `cancel` are compare-and-set operations from +`open` for both the upstream token span and each consumer obligation. +`transferring` becomes `transferred` only after the named replacement fence or +becomes `canceled` on mismatch/failure; duplicates are no-ops with diagnostics. +Required obligations becoming terminal advances +`obligationsTerminalEndSu`. That transition makes the cumulative ACK eligible; +it does not wait for ACK publication. The coalescer advances `ackQueuedEndSu`, +and only the later mux write callback advances `ackPublishedEndSu` and allows +record reclamation. A token span begins transfer only when the relay atomically +moves reconnect-grace ownership to a new token and the attach response +declares the new cumulative boundary, or cancels after +explicit/connection-close proof. The transaction creates the replacement +owner before marking source `transferring`; only exact recovery-complete +coverage terminally transfers it. + +| Event | Mandatory transition | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| spawn/attach response decoded | install provisional `sourceActivation` in synchronous `beforeResolve`; hold same-turn data privately; spawn commits after full operation validation, while reconnect transfers into an exact private recovery quarantine after contract validation and commits after fence/admission | +| invalid-checkpoint `restoreRequired` | keep the canceled relay record through response settlement; retire only the still-current record, wake capacity, and permit a fresh-token retry | +| model accepts; desktop/mobile parses | settle that consumer | +| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker | +| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting | +| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains | +| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof | +| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore | +| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage | +| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range | +| empty or non-empty recovery completion | require checkpoint-to-`recoveryEndSu` continuity, retain that live-start anchor after activation, then admit only an exactly contiguous live frame | +| exact exit during private recovery | wake the fence, reject later same-token admission, and publish exit only after the held prefix is admitted or exact cancellation proof retires it | +| recovery completion lacks writer capacity | retain range/fence state and retry on capacity; keep at most one admitted completion attempt | +| stale/overlapping recovery attempt | cancel only that attempt's token; never mutate the replacement mux, checkpoint, PTY ownership, physical process, or owner lease | +| current recovery cancellation publication/proof fails | after quarantine ownership transfer, close the exact provider generation and release its mux, provider, publishers, and registration once; retain physical PTY ownership for reconnect | +| partial remote cumulative byte ACK | advance/release exact byte credit; retain the covering source frame and obligation until a complete recorded boundary | +| contiguous higher-generation source token | keep the stream attached, bind later frames to the new identity, and preserve unsettled older-token frame mappings | +| remote snapshot replacement | reserve immutable span IDs/ranges, commit only after current-generation `SnapshotEnd` coverage, otherwise roll back | +| snapshot commit after proof reclaimed a span | reject commit without dereference; rollback removes the exact reservation and skips absent spans; later settle/detach prunes cached mappings | +| remote ACK-pending overflow snapshot | reserve covered spans, publish, commit exact snapshot identity/sequence, then trim; rollback and detach on failure | +| late remote detach after token cancellation | prune already-reclaimed mappings and detach idempotently without dereference or new credit | +| exit timeout receives cancellation proof | while generation-current, transfer projections, commit proof, prepare/finalize once, and close projection with generation checks between mutations | +| provider generation closes during exit proof | reject/remove barrier ownership; a late proof cannot commit, prepare, finalize, or publish exit | +| provider generations close over process lifetime | merge exact closed ranges; preserve out-of-order live gaps and reject every closed generation without one tombstone per reconnect | +| source tokens cancel repeatedly for one PTY | retain only the latest ordered canceled token for that PTY; replace at the next cancellation and clear at the next activation, PTY exit, or provider teardown | +| explicit live-token reset | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport | +| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof | + +There is no “abandon live token” transition. A main-side path that cannot prove +settlement, transfer, or relay cancellation must keep the ledger or close the +provider. Delayed renderer/mobile ACKs carry the consumer generation and cannot +settle a new provider generation or reused PTY ID. + +### State-machine pseudocode + +```text +session owner: + none --authenticated claim--> electing(next monotonic generation) + electing --session-grant write success--> active + electing --session-grant write error--> none(close generation) + electing --client close/cancel--> none + active --proved live replacement/tokens to grace--> + active(generation + 1, rotated lease) + active --unexpected client close--> grace + grace --proved resume--> active(generation + 1, rotated lease) + grace --30-second expiry--> none(invalidate lease, cancel grace cursors) + +relay token: + create -> activating + activating --response write success/no recovery--> active + activating --response write success/recovery--> recovering + activating --response write error--> closed(cancel generation) + recovering --completion not admitted--> recovering(retry on capacity) + recovering --completion admitted--> completion-in-flight(single) + completion-in-flight --write success--> active + completion-in-flight --write error--> recovering(exact replacement/cleanup) + recovering --invalid checkpoint--> restore-canceled-awaiting-response + restore-canceled-awaiting-response --exact response settles/current record--> + closed(delete record, wake capacity) + activating --stale/close--> closed(cancel) + activating|recovering|active --same-client replacement--> closing(superseded) + active --native exit--> sealed-unsettled + sealed-unsettled --exit write success--> sealed-unsettled(exit-published) + sealed-unsettled --exit write error--> closing(generation proof) + sealed-unsettled --relay applies suffix ACK/exact transfer--> closed + sealed-unsettled --timeout--> closing(exit-timeout cancellation proof) + active --cancel request--> closing + closing --cancel response/client close proof--> closed + +writer: + writable --write(false)--> saturated --drain--> writable + writable|saturated --close/error/detach--> closed + +main mux writer selection: + any queued liveness --> select liveness + control queued and (ordinary empty or controlsSinceOrdinary < 4) --> select control + ordinary queued --> select ordinary(reset controlsSinceOrdinary) + control queued after ordinary empty --> select control + drain --> writable(re-run the same selection; preserve FIFO inside each lane) + +main obligation: + open --consumer receipt--> settled + open --replacement created atomically--> transferring + transferring --exact replacement coverage fence--> transferred + transferring --missing coverage/replacement close--> canceled + open --relay cancel/close proof--> canceled + +main token span: + open --required consumers terminal--> obligations-terminal + obligations-terminal --coalescer queues cumulative end--> ack-queued + ack-queued --ACK write success--> ack-published(cleanup eligible) + ack-queued --ACK write error--> ack-queued(close/retry) + open|obligations-terminal|ack-queued + --attach boundary + replacement created--> transferring + transferring --exact recovery-complete fence--> transferred + transferring --gap/replacement failure--> canceled + open|obligations-terminal|ack-queued + --cancel response/client-generation close--> canceled + +main receive token: + unseen --beforeResolve--> provisional(cursor = checkpointSourceEndSu, heldData = []) + provisional --same-turn contiguous data--> provisional(append heldData; no projection) + provisional --all spawn/non-recovery attach/exit validation--> + active(commit lease, publish heldData in order) + provisional --validated recovery contract + private quarantine installed--> + receiving-activation(transfer lease to private sink, route heldData by range) + provisional --pre-transfer validation failure--> + rollback(drop heldData, restore active predecessor, request exact cancellation) + rollback --valid cancellation response--> prior exact state + rollback --publication/proof rejects--> fail operation(execution owner unavailable) + receiving-activation --quarantine exact checkpoint..recoveryEnd + complete fence--> + active(final commit, expectedLiveStart = recoveryEnd) + receiving-activation --exact exit--> + exit-sealed(admit held contiguous prefix or prove exact cancellation first) + active --first/next live starts at expectedLiveStart--> active(advance expectedLiveStart) + receiving-activation|active --gap/overlap/bad fence--> canceling-only-this-token + active --pty.exit--> exit-sealed + exit-sealed --prior model/projection/remote terminal--> exit-ack-queued + exit-ack-queued --runtime + renderer exit cleanup--> exited-awaiting-ack-publication + exited-awaiting-ack-publication --ACK success/generation close--> closed + exit-sealed --deadline--> acquiring-exit-cancellation-proof + acquiring-exit-cancellation-proof --proof + generation current--> + transfer-projection -> commit-proof -> prepare-once -> finalize -> close-projection + acquiring-exit-cancellation-proof --generation closes--> + closed(reject barrier; late proof has no authority) + acquiring-exit-cancellation-proof --proof cannot publish while current--> closing-provider + receiving-activation|active --overflow/cancel/close--> closed + +main provider-generation guard: + open(g) --close--> closed-ranges(add [g, g], merge adjacent/overlapping) + event(g in closed-ranges) --> reject(stale generation) + event(g in an unclosed gap) --> validate PTY incarnation normally + diagnostics --> closed range count + allocated-unclosed IDs below high-water + +main canceled-source retirement: + none(pty) --cancel token T--> latest(pty, T) + latest(pty, T) --later ordered cancel U--> latest(pty, U) + latest(pty, T) --frame T--> drop + latest(pty, T) --next activation/PTY exit/provider teardown--> none(pty) + +remote source ledger: + frame-open --partial cumulative byte ACK--> + frame-open(release exact byte delta; retain source mapping) + frame-open --ACK reaches encoded frame end--> frame-settled(settle exact source ranges) + old-token frame-open --contiguous strictly higher client+owner generation/new token--> + old-token frame-open + new-token receiving + receiving --stale/partial generation advance or identity discontinuity--> reject + +remote projection replacement: + live --reserve immutable spans + required SnapshotEnd seq--> transferring + transferring --current generation SnapshotEnd covers required seq--> transferred + transferring --stale/failure/disconnect--> live(rollback) + transferring --authoritative span already reclaimed--> + commit-rejected -> rollback(skip absent span, delete reservation) + live|transferring --authoritative token already canceled--> detached(prune cached spans) + +remote ACK-overflow recovery: + overflowed --reserve snapshot-covered spans--> transferring(single-flight) + transferring --snapshot publish + exact commit--> live(trim through snapshot seq) + transferring --serialize/publish/commit failure--> detached(rollback first) +``` + +```ts +function reserveSshAdmissionAtomically( + token: DeliveryToken, + frame: TokenizedPtyData +): SshAdmissionReservation { + const model = runtime.reservePtyData(frame) + let span + let projection + try { + span = token.ledger.reserveContiguous(frame, model) + span.require('model') + addConsumerObligationsFromCurrentPolicy(span) + projection = projectionLedger.reserve(toDesktopProjectionSpan(span)) + return { model, span, projection } + } catch (error) { + projectionLedger.rollbackIfReserved(projection) + token.ledger.rollbackIfUncommitted(span) + runtime.rollbackPtyData(model) + throw error + } +} + +function acceptSshFrame(frame: TokenizedPtyData): void { + const token = validateActiveProviderToken(frame) + let reservation + try { + reservation = reserveSshAdmissionAtomically(token, frame) + } catch (error) { + cancelTokenAfterAdmissionFailure(token, error) + return + } + const { span, modelReceipt, projectionSpan } = commitSshAdmission(reservation) + try { + projectionLedger.publish(projectionSpan, frame.data) + } catch (error) { + transferDesktopToModelRestore(span, 'renderer-send-failed', error) + } + fanOutToCurrentRemoteConsumers(span) + void modelReceipt.then( + (receipt) => { + span.settle('model', receipt) + commitPendingProjectionTransfers(span) + advanceObligationsTerminalEnd(token) + queueCumulativeAck(token) + }, + (error) => cancelTokenAfterModelFailure(token, span, error) + ) +} + +function retireDesktopDisplayPrefix(id: string, processedDisplayChars: number): void { + const selection = projectionLedger.reserveDisplayPrefix(id, processedDisplayChars) + const transaction = ledger.reserveDesktopSettlement(selection.ranges) + try { + validateDesktopSettlement(selection, transaction) + commitDesktopRangeAndLedgerAtomically(selection, transaction, 'renderer-parse') + } catch (error) { + ledger.rollbackDesktopSettlement(transaction) + projectionLedger.rollback(selection) + throw error + } + advanceObligationsTerminalEndForPty(id) + queueCumulativeAckForPty(id) +} + +function replaceDesktopProjection(id: string, reason: string): void { + for (const obligation of ledger.openDesktopObligations(id)) { + obligation.beginTransferTo('model-snapshot', reason) + } + scheduleModelRestoreAfterReceipts(id, reason) + commitTransfersWhoseModelReceiptsExist(id) + advanceObligationsTerminalEndForPty(id) // pending transfers remain ineligible + queueCumulativeAckForPty(id) +} + +function onAckWriteSettled(token: DeliveryToken, endSu: number, result: SinkWriteSettlement): void { + if (!result.ok) { + closeProviderAfterAckWriteError(token, result.error) + return + } + token.ackPublishedEndSu = Math.max(token.ackPublishedEndSu, endSu) + reclaimPublishedPrefix(token) + maybeCloseSealedDelivery(token) +} + +function onRelaySourceAck(record: RelayDelivery, ack: SourceAck): void { + requireExactDeliveryIdentity(record.identity, ack) + const pendingEnd = record.pendingSend?.span.sourceEndSu + if (ack.creditedEndSu > record.sentEndSu) { + require(ack.creditedEndSu === pendingEnd) + record.reservedAckEndSu = pendingEnd + return + } + applyCommittedBoundaryAck(record, ack) +} + +function retryRelaySourceSend(record: RelayDelivery): SendReservation | null { + const requiredEnd = record.reservedAckEndSu + return requiredEnd === null + ? reserveNextWindowSlice(record) + : reserveExactSameTokenBoundary(record, requiredEnd) +} + +function finishRecovery(candidate: RecoveryCandidate, fence: RecoveryComplete): void { + const expectedStart = candidate.checkpointSourceEndSu + const recoveredEnd = validateExactContiguousQuarantine(candidate.frames, expectedStart) + if (recoveredEnd !== fence.recoveryEndSu) { + cancelReplacementTokenOnly(candidate.identity, 'recovery-coverage-mismatch') + return + } + commitQuarantinedRecovery(candidate) + installLiveStartAnchor(candidate.identity, fence.recoveryEndSu) +} + +async function rollbackProvisionalActivation(lease: ReceiveLease): Promise { + lease.dropHeldDataAndRestoreActivePredecessor() + const proved = await cancelExactTokenThroughOriginatingMux(lease.identity) + if (!proved) throw executionOwnerUnavailable() +} + +function acknowledgeRemoteBytes(stream: RemoteStream, ackedEndByte: number): void { + const delta = stream.validateAndAdvanceCumulativeByteAck(ackedEndByte) + stream.byteBudget.release(delta) + for (const frame of stream.completeFramesThrough(ackedEndByte)) { + settleExactSourceRanges(frame.sourceRanges) + } +} + +function replaceRemoteConsumer(stream: RemoteStream, requiredSeq: number): void { + const reservation = remoteRanges.reserveImmutableSpans(stream, requiredSeq) + publishSnapshotEnd(stream).then( + (publication) => { + if (!remoteRanges.commitIfCurrentAndCovered(reservation, publication)) { + remoteRanges.rollbackSkippingReclaimed(reservation) + } + }, + (error) => remoteRanges.rollback(reservation, error) + ) +} + +async function cancelTimedOutExit(exit: ExitBarrier): Promise { + const proof = await requestCancellationProofWithoutMutation(exit.identity) + if (!exit.isCurrentGeneration()) return + transferPublishedProjection(exit.identity) + commitCancellationProof(proof) + prepareExitOnce(exit.identity) + if (!exit.isCurrentGeneration()) return + finalizeExit(exit.identity) + if (!exit.isCurrentGeneration()) return + closeProjection(exit.identity) +} + +async function recoverAckOverflow(stream: RemoteStream): Promise { + const snapshot = await serializeBudgetedSnapshot(stream) + const reservation = reserveCoveredRemoteSpans(stream, snapshot.seq) + try { + require(publishSnapshotFrames(stream, snapshot).published) + require(commitExactSnapshotReplacement(reservation, snapshot.source, snapshot.seq)) + trimQueuedOutputThrough(stream, snapshot.seq) + } catch (error) { + rollbackSnapshotReplacement(reservation, error) + detachRemoteStream(stream) + } +} +``` + +Token/span validation, charged admission, and every membership obligation occur +before asynchronous model execution. Desktop/remote fan-out may begin after +the admission budget owns the frame, but the model obligation settles only +after the emulator callback. Admission wait leaves it open and pauses reads at +the charged limit; rejection triggers token cancel rather than acknowledging +data the model did not own. Renderer progress is always display-side input to +the range queue and is never interpreted directly as source units. + +## Incremental frame decoding + +The actual main transport contract is extended as follows: + +```ts +type MultiplexerTransport = { + write(data: Buffer, onSettled: (result: SinkWriteSettlement) => void): SinkWriteResult + writableLength(): number + writableHighWaterMark(): number + onWriteDrain(cb: () => void): () => void + onData(cb: (data: Buffer) => void): () => void + onClose(cb: () => void): () => void + pauseReads(): void + resumeReads(): void + close(): void +} +``` + +`waitForSentinel` constructs these hooks from the real `ClientChannel`: +`channel.stdin.write(buf, callback)`, writable length/high-water mark, and +`on('drain')` for output, plus +`channel.pause()/resume()` for relay stdout. This covers ssh2 and +`SystemSshCommandChannel`, whose stdout-facing channel implements the same +Readable contract. WSL child stdin/stdout exposes the same metrics, +callback/drain, and pause/resume adapter. The startup residue is delivered once +before later data. + +`SshChannelMultiplexer` also owns one main-to-relay writer with exactly three +bounded FIFO lanes: + +1. `liveness` for the coalesced probe, including the one saturated-epoch + bypass; +2. `control` for source ACK, cancellation, exit, RPC request, and response + frames; +3. `ordinary` for `pty.data`, including PTY input/paste traffic. + +The scheduler rechecks priority after every frame and whenever drain reopens +the writer. Liveness wins first. Control wins over queued ordinary traffic +until four control frames have been selected while ordinary remains queued; +the next selection must be one ordinary frame and resets the counter. FIFO is +preserved within each lane. All non-bypass writes still wait for their +per-write callback and `onWriteDrain` after saturation. Producer-side bulk +shaping and fixed filesystem-frame admission remain with their existing +owners; they do not create a fourth mux-writer lane. Thus control cannot remain +behind an old ordinary backlog after drain, ordinary input cannot starve under +a sustained control stream, and V1 ACK settlement has a concrete write fence. + +On the relay, the initial client uses `process.stdin.pause()/resume()` and each +accepted Unix-socket/named-pipe client uses `socket.pause()/resume()`. +`attachClient` accepts those hooks; `detachClient` releases them. Handshake +decoders retain synchronous first-frame behavior and transfer both exact +residue and pause ownership to the dispatcher. + +Both protocol copies then use this state machine: + +1. `feed()` appends buffer views without copying and charges bytes. +2. Drain at most 64 frames or 4 ms. +3. If another complete frame remains, acquire one idempotent decoder pause + epoch and continue with `setImmediate`. +4. Release the pause only when complete-frame backlog and queued bytes fall + below low water. +5. `reset`, handshake `drain`, transport close, and dispose cancel the + continuation and release exactly the epoch they own. + +Main model-pressure mode is the one bounded exception to immediate read pause. +It may classify complete frames into the charged 1 MiB/64-frame data +quarantine while dispatching only eligible control/liveness frames, then +either drains data and token-ordering fences in original order or closes the +provider at the deadline. Exit, recovery completion, and activation cannot +overtake a preceding quarantined frame for the same token. It does not scan +past an incomplete or oversized frame and cannot expand the ordinary decoder +input cap. + +Handshake consumers require the first complete frame synchronously. Preserve +that behavior, then yield before later frames. `drain()` must cancel a scheduled +continuation and return all unread handshake residue in exact byte order so the +next consumer receives it once. + +The decoder permits one advertised valid 16 MiB frame plus 1 MiB input slack. +A partial valid frame keeps reads enabled until that cap; exceeding it closes +the offending transport rather than retaining back-to-back maximum frames. +Oversized payloads are discarded incrementally without full retention. +Sender-side finite PTY bursts and the reserved control lane bound how far a +keepalive/control frame can sit behind PTY data. While paused by this decoder, +the mux suspends both dead-link conjuncts. Resume rebases `lastReceivedAt` and +all outstanding header-ACK timestamps exactly like the existing wake-gap +handler; outgoing keepalives still run. + +In `SshChannelMultiplexer.handleFrame`, clamp header `ack` to +`nextOutgoingSeq - 1` and delete only keys already present in +`unackedTimestamps`. Never iterate a dense integer range toward an untrusted +32-bit value. This remains independent from PTY credit. + +## Cleanup and reconnect + +Every dispatcher client has a generation, and the negotiated session owner has +a separate transferable owner generation. Socket close/error, +`attachClient`/`detachClient`, and dispose invalidate or advance the client +generation atomically. Unexpected disconnect retains the owner generation and +lease through grace; only a proved replacement increments it, and grace expiry +invalidates the lease so a later election starts a new generation. Each +transition: + +- cancel read-pause epochs, writer callbacks, and scheduled work; +- reject queued liveness, control, and ordinary mux frames, clear lane heads, + and reset the four-control fairness counter; +- roll back uncommitted receive-activation leases to their exact prior + active predecessor, drop lease-private data, and await cancellation of only + the provisional token through its originating mux; +- after a validated recovery contract transfers the lease, retain held frames + only in that attempt's bounded quarantine and let cancellation proof or + provider-generation close own failure cleanup; +- on exact exit, seal the private lease, wake its fence, and either admit the + held contiguous prefix or validate cancellation against the highest exact + frame observed before retention/restore decisions; +- fail closed on the exact provider generation when a current recovery cannot + publish cancellation or validate its proof; provisional spawn rollback + failure instead surfaces execution ownership as unavailable; +- retain canceled `restoreRequired` relay deliveries until their response + settlement callback, then delete only the exact still-current record; +- clear or transfer one recovery-completion attempt and its capacity listener; +- move active PTY cursors to reconnect grace or cancel them; +- preserve sealed-unsettled suffixes until the relay applies their ACK, exact + transfer, or cancellation/generation-close proof; +- retry or cancel queued cumulative ACK state without collapsing + `obligationsTerminalEndSu`, `ackQueuedEndSu`, and `ackPublishedEndSu`; +- retain or cancel an exact early-ACK send boundary with its owning token; +- roll back uncommitted desktop reservations and transfer committed + projection IDs before clearing renderer queues; +- cancel uncommitted terminal-fact publications, restore their prior scanner + snapshots, and transfer committed projection scanner state exactly once; +- roll back any remote snapshot replacement before clearing ACK-overflow state; +- preserve partial remote byte watermarks while retaining covering source + frames, and prune cached mappings whose authoritative spans were reclaimed; +- reject and remove exit barriers plus cancellation/preparation ownership on + provider-generation close so late proofs cannot publish final exit; +- close each per-generation model-migration gate after its settlement, remove + its per-PTY fence and timer exactly once, and retain any earlier outstanding + per-PTY fence across an overlapping reconnect; +- on migration-owned callback failure, release only that PTY's admission + charge, timer, and tracked completion before reset/restore; do not close the + shared provider or sibling admissions; +- replace the latest canceled-token record per PTY on ordered cancellation and + clear it on exact next activation, PTY exit, or provider teardown; +- release shared spans no longer referenced; +- recompute native PTY pause state; +- expire pending RPC ownership. + +Unexpected loss of the session-owner generation with active PTY tokens creates +one `reconnect-grace` cursor per PTY for 30 seconds. It retains at most +512 Ki su and 2 MiB charged bytes per PTY; reaching either cap pauses that PTY +instead of dropping more output. A valid owner resume atomically installs the +new owner generation, transfers each cursor to a new token, and returns only +activation and recovery-range metadata. Source-ranged `pty.data` then drains +before the completion fence and live activation. Expiry invalidates the owner +lease, +emits `pty.deliveryCanceled` when possible, releases the gap, resumes the PTY, +and records restore-required/data-gap telemetry. Ordinary subscriber loss or +no-subscriber state outside owner grace retains only the existing replay tail. + +`SshRelaySession.reattachKnownPtys` waits for session-grant readiness and each +PTY's model-migration fence, then runs at most eight attaches concurrently. +Each PTY has its own 10-second attempt deadline and `try/catch`; one failure +cannot abort later PTYs. `notFound` and identity mismatch keep their existing +stale-lease behavior. Other errors are recorded, retried once with bounded +jitter while the same reconnect attempt is current, then surfaced per PTY +without tearing down successful siblings. Time-to-last-reattach is therefore +bounded by waves rather than `N × RTT`. + +Reconnect and replay always create new tokens even when IDs are reused. Old +ACKs and callbacks fail generation/token checks. The attach response declares +the exact checkpoint and recovery end; ordinary source-bearing `pty.data` +proves the gap and the completion fence establishes the live boundary, so main +never double-ingests old obligations. Replace +`SshRelaySession.forwardReattachReplay` with this tokenized intake and delete +`RECONNECT_REPLAY_DUPLICATE_WINDOW_MS`/`shouldForwardReattachReplay`: a +wall-clock fingerprint can suppress legitimate identical output, while the +source checkpoint is authoritative. Exit before reattach may still synthesize +`code: -1` after `notFound`; retaining remote exit tombstones is a separate +behavior change and is not required for the memory bound. + +Recovery receipt is transactional. Main retains quarantined recovery `pty.data`, +the candidate mux, token, checkpoint, and `recoveryEndSu` until the exact +completion fence validates. It commits no partial body to the model. On +failure it requests cancellation only through the candidate mux and accepts +only matching proof; a stale attempt may not touch current mux state. A +quarantine-only attempt may not yet have a main-intake token identity, so +`SshRelaySession` first requires `canceled: true`, finite safe cumulative +ends, `creditedEndSu` equal to the recovery checkpoint, and `sentEndSu` +covering the highest exact-token source range observed privately. Only then is +the intake's no-identity application an intentional no-op; malformed, +under-covering, or checkpoint-mismatched proof fails closed. A +current attempt whose cancellation publication or proof rejects closes the +exact provider generation and its registered resources once, rather than +leaving ambiguous authority. After a successful empty recovery it keeps +`recoveryEndSu` as a live continuity anchor until the first live frame arrives. +A proved recovery failure marks the existing PTY lease detached and retryable +rather than deleting ownership or terminating the physical process. + +For an operation-ID spawn replay, physical result lookup completes first; the +current request then creates and fences a new subscription as specified above. +Disconnect after physical commit/before response leaves the PTY retained but +no stale token. + +PTY shutdown and natural exit cancel native pause state before disposing +node-pty. Natural exit may still leave a charged sealed-unsettled delivery +record; only its native process is gone. Relay disposal generation-closes those +records and clears scheduler/exit timers before walking PTYs. A folder +workspace removal follows the same PTY teardown path; no design step assumes a +Git worktree. + +### SSH producer-pause intent + +Negotiated V1 implements `SshPtyProvider.pauseProducer` and `resumeProducer` +with a token/generation-scoped `pty.setDeliveryPaused` request; window size is +not renegotiated to zero. Relay marks that owner delivery ineligible and pauses +the native PTY only when all required deliveries are ineligible, while another +healthy subscriber may continue. Resume restores eligibility under the +original window. `setPtyBackgrounded` is a separate token-scoped scheduling +hint; it does not itself drop source data or silently convert a lossless model +obligation into keep-tail behavior. + +For V1, desktop pending-cap pressure transfers the recoverable projection and +does not call producer pause. Model-admission or required-lossless pressure and +explicit background policy may call it. Local, daemon, and legacy provider +behavior remains unchanged. Update the provider-interface and IPC comments +that currently assume SSH has an independently bounded pending queue. + +## Budgets + +Every limit has one unit. Source-flow limits use `su`; heap/transport limits use +bytes measured at the point that owns the memory. + +| Resource | High limit | Low/flush point | Action | +| ------------------------------- | -----------------------------------------: | ------------------------: | ------------------------------------------- | +| Legacy publications per client | 2 MiB | 1 MiB | pause affected PTYs; release on drain | +| Legacy publications per relay | 32 MiB | 24 MiB | pause affected PTYs | +| Legacy producer-held PTY frame | 128 KiB | next admission | hold one; pause before another | +| V1 data frame | 16 Ki su target | — | scalar-safe slice/coalesce | +| Encoded PTY frame | 128 KiB | — | reduce source slice before publish | +| Token outstanding credit | 256 Ki su | 64 Ki su newly freed | stop send / eager ACK | +| Sealed-unsettled suffix | token window; 30 s timer per subscriber | ACK/transfer/cancel proof | retain ledger; cancel stalled token | +| Recovery `pty.data` | 16 Ki su target | next turn | normal source lane before completion | +| Reconnect-grace source | 512 Ki su per PTY | — | pause at cap | +| Retained live data per PTY | 2 MiB | 1 MiB | pause source owner | +| Retained live data per relay | 64 MiB | 48 MiB | pause affected source owners | +| Replay tail | existing 100 Ki su × 50 PTYs | — | source-range trim; bytes charged globally | +| Liveness/control writer reserve | 25% of high water, capped at 64 KiB | drain | producer lanes cannot consume | +| Minimum PTY sink capacity | 8 KiB non-reserved empty | — | reject PTY subscription | +| Metadata control response | 64 KiB | — | body must use producer stream | +| Control queue | 256 frames / 1 MiB | drain | close subscriber; owner reconnect | +| Reshapable bulk producer frame | current non-reserved sink capacity | next frame | slice, admit, recheck priority | +| Fixed filesystem frame | existing 256 KiB raw payload | empty producer/sink epoch | admit one unchanged, then await settlement | +| Decoder | 16 MiB frame + 1 MiB slack | no complete backlog | pause/close at hard cap | +| Decoder turn | 64 frames or 4 ms | next turn | `setImmediate` | +| Header-ACK timestamp entries | 4095 + 1 reserved | ACK/resume | stop ordinary writes; coalesce liveness | +| Main receive-activation hold | 256 Ki su / 2 MiB per token; 64 MiB total | recovery completes | pause aggregate; close provider on cap | +| Main model admission per PTY | 256 Ki su / 2 MiB | 128 Ki su / 1 MiB | pause mux admission | +| Main model admission global | 12.5 Mi su / 64 MiB | 8 Mi su / 48 MiB | pause mux admission | +| Main blocked-intake slot | one 128 KiB frame | model low water | hold blocked frame | +| Main pressure control reserve | 1 MiB / 64 frames | 10-second close deadline | quarantine data; service control | +| Main model migration fence | 10 seconds | — | reset generation / restore required | +| Main desktop in-flight | existing 512 Ki su per PTY / 8 Mi su total | existing lows | transfer/restore policy | +| Lossless remote send ledger | 2 MiB/stream, 16 MiB global | 1 MiB/12 MiB bytes | stop send; explicit detach transfers/closes | +| Activation/exit/owner grace | 30 seconds | — | token cancel/gap policy | + +Retained strings are charged once as +`max(Buffer.byteLength(value, 'utf8'), 2 * value.length) + 128`; encoded buffers +use exact `Buffer.byteLength`; ledger/cursor entries charge 128 bytes. Shared +strings are not multiplied by subscriber count, but each cursor is charged. +Sent-but-uncredited source units are not described as retained memory. + +The table is arithmetically reachable: + +- 256 Ki ASCII su charges at most about 512 KiB plus records. +- 256 Ki BMP/CJK su at three UTF-8 bytes each charges about 768 KiB. +- 256 Ki su of surrogate-pair characters has 128 Ki scalar values and about + 512 KiB UTF-8/UTF-16 storage. +- A 512 Ki-su ordinary reconnect backlog is at most about 1.5 MiB for valid JS + strings; 32 16-Ki-su span records keep it below the 2 MiB per-PTY cap. + Highly fragmented transformed records bind the charged 2 MiB cap earlier. + Recovery is streamed and never compared with the 1 MiB control queue. +- JSON control-character escaping can reach six encoded bytes per su, so a + 16 Ki-su frame is below 96 KiB plus envelope and the separate 128 KiB cap. +- A transformed span with `rawLength !== data.length` consumes source credit + by `rawLength` and heap/wire budgets by its actual retained/encoded bytes; + neither value is converted into the other. +- Fifty PTYs cannot each reach 2 MiB because the 64 MiB relay cap binds first; + the independent 48 MiB global low water prevents immediate resume thrash. +- Model admission charges captured strings and queue records before a Promise + link owns them. Its 64 MiB global high water matches the relay retained-data + cap and its 12.5 Mi-su high equals 50 token windows, while each + 256 Ki-su/2 MiB per-PTY high covers one full token window; the separate + 128 KiB blocked slot covers the decoder frame that triggered pause without + hiding it in the model budget. +- Encoded lossless-remote bytes remain charged until full frame-boundary ACK or + atomic transfer; the ledger's source units are never estimated from bytes. + +One re-entrant native callback after `pause()` is a charged transient overshoot. +The callback may cross a retained-memory high cap by at most one charged native +chunk, but it is never admitted past token credit: splittable source is trimmed +to remaining window and an indivisible transform waits. If retained memory +crosses a hard cap, stop publication, mark restore-required, and retain only +the already-owned bounded state. Module constants allow test overrides; Linux +Docker heap/RSS plateaus validate allocator slack before release. + +## Telemetry + +Expose aggregate counters in existing relay/main diagnostic snapshots without +logging terminal contents or raw PTY IDs: + +- active/activating/recovering/closing tokens, owner generation/election/resume + outcomes, and PTY-subscribing clients; +- paused and reconnect-grace PTYs, grace bytes, recovery/fence latency, exact + gap ranges, completion retries, expiries, and gap outcomes; +- client writer state, lane depths, reserved-byte denials, and activation-fence + latency/cancellation, split bulk bytes, and per-priority wait; +- current/peak retained bytes and outstanding su by redacted client/PTY; +- pause count and total paused milliseconds; +- sink saturation count and duration by stdout/socket; +- slow-client detach and control-overflow counts; +- ACK accepted, duplicate, regression, over-credit, malformed, and stale-token + counts, early-boundary reservations/retries, plus ACK frames/second, + entries/frame, and encoded bytes/second; +- span obligations opened, transferring, settled, transferred, canceled, + duplicate-terminal, and oldest-open/oldest-transferring age by consumer + class; +- per-token obligations-terminal, ACK-queued, and ACK-published ends plus their + deltas; sealed-unsettled count, suffix size/age, exit-published state, and + timeout cancellation proof; +- desktop projection reservations, commits, rollbacks, model transfers, stale + identity rejects, and outstanding source/display ranges; +- model-admission current/peak bytes per PTY/global, blocked duration, + emulator completion/failure/cancellation, migration-fence latency/timeout, + late-generation rejects, and low-water resumes; +- lossless-remote encoded bytes, partial/frame-boundary ACKs, mapped source + ranges, generation rejects, ACK-overflow snapshot reserve/publish/commit/ + rollback, and transfer/cancel outcomes; +- decoder queued bytes, yielded turns, maximum frames/turn, and maximum + callback duration, read-pause duration, liveness-timeout suppression, + header-ACK timestamp depth/cap denials, and keepalive coalescing; +- exit-barrier timeout count; +- closed provider-generation range count and active gaps; +- reconnect attach wave latency, per-PTY retry/failure, token supersession, + relay-initiated cancellation proof, and time-to-last-success; +- same-build V1 grants, capability omissions from legacy clients, + method-not-found fallbacks, unsupported manual version skew, and discovered + orphan prior-version daemons, keyed by relay build/version rather than + legacy ACK traffic. + +Rate-limit warnings by connection and reason. Log thresholds and state +transitions, never data. Add an E2E-only snapshot request so tests can assert +plateaus, obligation conservation, pause/resume, and zero false reconnects +without parsing logs. + +## Incremental implementation map + +This PR implemented the design as the independently testable layers below, +without replacing local/daemon protocols or claiming live evidence for every +deployment topology. The slices remain useful review boundaries even though +they ship in one PR. + +### Slice 1: hostile-header and accounting hardening — implemented + +- In `src/main/ssh/ssh-channel-multiplexer.ts`, clamp header ACKs and delete + only present timestamp keys; cap retained timestamps. +- Add focused unit tests for `ack=0xffffffff`, concurrent ACK/timeout cleanup, + and liveness rebasing. +- No relay, daemon, runtime, or provider protocol change. + +### Slice 2: bounded decoder turns — implemented + +- In both relay protocol decoders, bound frames and time per turn, preserve a + synchronous first handshake frame, pause reads during continuation, and + transfer exact residue on reset/drain. +- Adapt ssh2, system SSH, WSL child stdio, relay stdin, Unix sockets, and + Windows named pipes behind their existing transport constructors. +- No PTY credit or session negotiation; prior-version peers keep the same + frame format. + +### Slice 3: one drain-aware legacy writer — implemented + +- Add a concretely named dispatcher writer owning ordinary, bulk, control, + sentinel, residue, and `--connect` bridge bytes. +- Route existing `RelayDispatcher.notify`, `notifyBulk`, fs/Git producers, and + `runConnectMode` through its bounded lane admission; stop after + `write(false)` until callback/drain. +- Preserve fixed `fs.streamChunk` size/offset compatibility in an exclusive + empty-sink `fixed-bulk` lane; slice only reshapable bulk and PTY output. +- Because legacy `notify` is synchronous and broadcast, add a bounded + transport-only publication record containing one encoded frame and the set + of client generations that have not accepted it. A client returning + `write(false)` leaves that set because Node accepted the frame; a saturated + client is retried only for that client, and close cancels its membership. + This is delivery bookkeeping, not source credit. +- Return producer admission to `PtyHandler`. Until a publication is admitted, + retain it in the existing PTY pending-output owner; at its hard byte cap + pause node-pty, then resume below low water. Cap per-client and aggregate + publication bytes. Before Slice 5a establishes authenticated roles, treat + every subscribed connection equally: any retained subscriber may pause that + PTY, and only transport close cancels its membership. Do not infer or detach + an “additional” client by constructor or connection order. Authenticated + slow-subscriber eviction begins only after Slice 5a. Never silently drop a + legacy PTY frame. +- Preserve the current 8 ms/1 ms PTY cadence and current replay semantics. The + replay tail stays the chunked `RecentPtyOutputBuffer` added in + `79ec57d04`; `PtySourceReplayIndex` now indexes it without replacing it with + the former per-append rolling string. +- This slice bounds relay writable queues for legacy main, WSL, direct SSH, + and detached relay socket/named-pipe clients without changing the wire. The + separate local terminal daemon is not exercised or changed. + +### Slice 4: bounded main SSH intake — implemented + +- Make `SshRelaySession.wireUpPtyEvents` hand each event exactly once to a + main-only intake under `src/main/ipc/`; SSH no longer bypasses the existing + pending-output authority. +- Meter model admission and its emulator callback chain, reserve before + capture, pause decoder reads at the cap, and return completion/failure + receipts. +- Introduce transactional projection admission with a legacy immutable + admission ID, provider generation, PTY incarnation, display interval, and + existing sequence/`rawLength`; it has no V1 token or source-credit + coordinate. Slice 5b extends that record to `DesktopProjectionSpan`. Keep + local and daemon providers on their existing intake until a shared migration + is separately justified. +- Preserve current main's ordered terminal side-effect facts and dropped-output + DEC mode 2031 scanner snapshots across admission, salvage, rollback, transfer, + and explicit gaps. Preserve `terminal-stream-byte-length.ts` UTF-8 accounting + and flush boundaries for remote streams; never reuse it as source credit. +- Extend the existing Docker ACK-stall test only for behavior this slice + implements: direct SSH/deployed relay/desktop intake and active typing. + +### Slice 5a: SSH session semantics — implemented for SSH + +- Add the transport-neutral `PtyConsumerSession` state machine and only the SSH + readiness adapter decision above. +- Implement authentication, generation, capability intersection, activation + fence, close cleanup, exact sentinel/residue transfer, and legacy fallback. +- Carry the endpoint credential through split-SFTP per-launch namespace + mapping, with secure canonical shell-path generation when no marker exists. +- Local in-process, daemon hello, and remote-runtime adapters reuse semantics + only when their own changes need it; they are not prerequisites for SSH V1. + +### Slice 5b: direct SSH source credit — implemented + +- Add the relay immutable span ledger/scheduler and tokenized cumulative ACKs + for one direct SSH desktop/model consumer path. +- Return exact `sourceActivation` metadata and install its provisional receive + lease synchronously in mux `beforeResolve`; commit only after provider and + exit-race validation. +- Reserve reentrant ACKs only at the exact pending same-token boundary and + retry that boundary after failed send settlement. +- Reuse Slice 4's projection identities and transactional admission. Separate + obligations terminal, ACK queued, and ACK published state from the first + implementation. +- Offer V1 on every new session while retaining bounded legacy compatibility + for capability-omitting clients and method-not-found relays. + +### Slice 5c: exit, reconnect, and replay — implemented + +- Implements sealed-unsettled exit, cancellation proofs, owner reconnect grace, + token replacement, and exact transfer. +- Carries recovery as contiguous source-bearing `pty.data` from checkpoint to + recovery end; fence live output with single-flight, capacity-retryable + `pty.recoveryComplete` writer settlement. +- Adds a bounded source-range index beside the current + `RecentPtyOutputBuffer`, preserving its append performance and legacy + output. +- `SshPtyOutputModelMigration` freezes exact generation/PTy admission, + `SshRelaySession` awaits the returned per-PTY Promise before constructing + `sourceRecovery`, and `OrcaRuntimeService` detaches only the failed PTY's old + headless model before snapshot-backed restore. +- `SshPtyModelAdmission` retains generation-fatal callback handling except when + the exact key is actively migrating; `SshPtyOutputIntake` uses the same key + to keep provider close from escaping that migration owner. +- Reconnect always re-offers V1; its late-ACK, timeout, supersession, and + generation-close deterministic oracles pass. + +### Slice 5d: required remote consumers — deterministic seam implemented + +- Map remote `terminal.multiplex` encoded-byte ACK frames to accepted source + ranges without changing its transport authentication or granting it an SSH + owner lease. +- Add `ackOutputSourceRanges: 1` and an opaque echoed stream generation; + preserve `ackOutput: 1` delta semantics for old clients and best-effort + streams. +- Make ACK-overflow snapshot recovery reserve, publish, commit, then trim; + rollback before detaching on any failure. +- Keep mobile notification replay epochs/watermarks outside terminal stream + identity and ACK mapping; neither can settle a PTY source range. +- Extend the extracted `terminal-output-frame-chunks.ts` seam with immutable + composite mapping input while preserving `694363805`'s code-unit scanner, + allocation profile, sequence rounding, and equivalence benchmark. +- Live headed paired-server and headless `orca serve` validation remains + required before claiming those topologies; neither is inferred from Docker + SSH. +- Keep best-effort remote streams outside upstream obligations. + +### Slice 5e: adversarial lifecycle reconciliation — implemented + +- Retain canceled invalid-checkpoint deliveries through exact + `restoreRequired` response settlement, then retire only the current record. +- Fail closed on the exact provider generation when a current recovery + cancellation cannot publish or its proof rejects; release generation-owned + resources once without clearing physical PTY ownership. +- Allow partial cumulative remote byte ACKs to release byte budget without + settling an incomplete source frame, and admit only contiguous strictly + higher client/owner generations with a new token. +- Make encoded snapshot replacement commit idempotent; reject reclaimed-span + commits and make rollback, settle, and detach prune absent spans safely. +- Split main-to-relay writes into liveness/control/ordinary FIFO lanes, select + control first after drain, and guarantee ordinary progress after four + controls. +- Hold provisional activation data until provider/claim/exit validation + commits for spawn, or until a validated reconnect contract installs its + private quarantine; rollback owns pre-transfer failure, while cancellation + proof or generation close owns post-transfer failure. +- Acquire exit cancellation proof before projection transfer and proof commit; + prepare/finalize once and fence every post-proof step by provider generation. +- Keep timeout cancellation authority operation-local and compact monotonic + closed provider generations into exact merged ranges without weakening stale + event rejection. +- Seal private recovery on exact exit, retain rollback authority through a + stale transfer, and include every exact-token private frame in the + cancellation-proof watermark. +- Replace process-lifetime per-token blocking state with one latest ordered + cancellation record per relay PTY, cleared only at an exact lifecycle + boundary. + +### Slice 5f: remaining topology validation and adapters — deliberately deferred + +- Local in-process and daemon hello remain unchanged; neither adopts SSH + framing or source credit in this PR. +- WSL, Windows ConPTY/named-pipe, local daemon/provider, folder workspace, + mixed-version, and prior-version-orphan live evidence remains separate. +- Promote reliability-gate provider coverage only for topologies with recorded + executable evidence. + +Keep new lifecycle ownership out of already-large modules where a concrete +domain file is clearer. Do not add max-lines disables, generic helper modules, +native dependencies, or a per-file lint exception. + +## Tests + +The checklist below is the normative matrix, not a claim that every physical +topology or the current shared-worktree diff ran. Implemented seams and prior +recorded evidence include: + +- shared session negotiation, grant publication, stale-owner recovery, source + ledger, ACK queue/publication separation, and cancellation proof contracts; +- relay writer admission/drain/liveness, zero-display source publication, + operation-ID retry activation, fixed-size filesystem compatibility, + capacity-aware Git/PTY slicing, decoder caps, and token rotation; +- main model admission, desktop identity/rollback, exit barriers, remote + reserve/commit/rollback, token cancellation, exact recovery continuity, + eight-wide reconnect, retention budgets, unconditional negotiation, and + deployment; +- current-working-tree restore-response retirement, fail-closed recovery cancellation, + partial remote ACK, higher-generation token rotation, reclaimed-span + idempotence, three-lane mux fairness, provisional claim gating, and + generation-fenced exit proof ordering plus bounded exact generation closure; +- the four-test Docker OpenSSH/deployed-relay suite for source-window plateau, + concurrent typing, fixed-size filesystem/Git churn, and owner reconnect. + +The earlier rebased checkpoint at +`38fbf3520741ce4ccd86410f7cea9ff8217a7834` passed: + +- frozen dependency setup, full lint, full typecheck, `git diff --check`, and + the 52-gate reliability manifest validation; +- all Linux, macOS, Windows, and WSL relay build targets plus + `electron-vite build --mode e2e`; +- 24 focused SSH/source-credit files with 354 tests; +- 14 watcher/daemon/package/glibc-adjacent files with 409 tests; +- a 57-file deterministic changed-surface sweep with 1,186 tests; +- the joined renderer/headless snapshot seam with 59 tests and its broader + focused slice with 180 tests; +- the relay recovery and fixed-size filesystem/Git slice with 105 tests. + +That checkpoint's Docker OpenSSH/deployed-relay run passed all four tests in 1.0 minute: +direct typing median/worst was 107.7/113.6 ms; ACK-stalled typing was +3.6/107.3 ms at an exact 262,144-source-unit plateau; fixed-size filesystem/Git +churn was 148.1/161.1 ms with 93 bulk reads; owner reconnect completed in +15.7 seconds. These measurements are direct SSH/deployed Linux relay evidence +only. + +The four-case Docker results predate the final adversarial lifecycle +reconciliation and are not current-head proof for those races. On the earlier +working tree, a deterministic test reproduced recovery completion becoming +visible at source unit 4 while lease-held recovery ended at 8. After the +validated contract began private quarantine ownership before awaiting the +body, seven provider/session files passed 94 tests and the isolated deployed +Linux relay/direct-SSH reconnect case passed in 17.7 seconds with 256 recovery +frames plus post-reconnect terminal/filesystem proof. Against the same +private-transfer artifact, the final four-case run passed in 1.1 minutes: +direct typing was 6.2/7.4 ms median/worst; ACK-stalled typing was 5.5/16.3 ms +at exactly 262,144 held source units; fixed-size filesystem/Git churn was +141.2/151.5 ms with 95 bulk reads; reconnect completed in 16.0 seconds. All +other live topologies remain separate. + +After the exact-PTY migration-failure containment fix and current-main merge, +`electron-vite build --mode e2e` and `pnpm run build:relay` produced the exact +artifact hashes recorded above. The four-case Docker command then passed in +56.7 seconds without a source-credit override: direct typing was 5.3/108.9 ms +median/worst; ACK-stalled typing was 4.8/107.6 ms at exactly 262,144 held source +units; fixed-size filesystem/Git churn was 152.0/173.3 ms with 84 bulk reads; +reconnect completed in 14.3 seconds. This is direct SSH/deployed Linux relay +evidence only. + +At the implementation commit above, the source/intake slice passed 12 files and 205 +tests, the provider/session slice passed 16 files and 192 tests, and this +complementary lifecycle slice passed 13 files and 150 tests: + +```bash +pnpm exec vitest run --config config/vitest.config.ts \ + src/relay/relay-pty-source-restore-retry.test.ts \ + src/main/ssh/ssh-relay-session-recovery-races.test.ts \ + src/main/runtime/rpc/terminal-source-range-ledger.test.ts \ + src/main/runtime/rpc/terminal-multiplex.test.ts \ + src/main/ssh/ssh-multiplexer-transport-writer.test.ts \ + src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts \ + src/main/providers/ssh-pty-notification-routing.test.ts \ + src/main/providers/ssh-pty-source-delivery-ledger.test.ts \ + src/main/ssh/ssh-pty-retired-source-deliveries.test.ts \ + src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts \ + src/main/providers/ssh-pty-provider-exit-race.test.ts \ + src/main/ipc/ssh-pty-output-exit-deadline.test.ts \ + src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot +``` + +The deterministic recovery seam specifically covers an empty recovery followed +by a gapped live frame, overlapping reconnect attempts, late frames after +cancel, exact exit during private recovery, stale-transfer rollback authority, +capacity-rejected and post-restore proof watermarks, typed stale-owner retry, +response-settled `restoreRequired` retirement, completion arriving before +lease-held body frames, and fail-closed exact-generation cleanup. A separate +10,000-rotation oracle proves canceled-token retirement remains one record per +PTY. Transferred recovery data remains private until body/fence validation, +and a proved token-local failure causes no physical PTY shutdown, ownership +delete, or lease expiry. +The model-migration failure oracle uses two PTYs in one provider generation: +one raw callback rejects under an active fence, returns +`checkpointUnavailable`, resets one model, and releases its charge and timer; +the sibling checkpoint, provider mux, filesystem/Git transport, and another +provider generation remain live. A paired admission oracle proves the same +failure without an active migration still closes the generation and rejects +its sibling receipt. +The remote-consumer seam covers partial cumulative ACK, safe higher-generation +token rotation, and token cancellation followed by late detach/replacement +commit, proving reclaimed span IDs reject commit, preserve rollback ownership, +and are then cleaned up idempotently. + +### Normative unit and property matrix + +- accept monotonic cumulative ACKs and reject duplicates, regressions, + over-credit, invalid numbers, wrong clients, wrong PTYs, and stale tokens; +- prove with generated span/ACK sequences that + `sentEndSu - creditedEndSu <= windowSu`; at `window - 1`, slice a + splittable frame to one source unit and hold an indivisible transform; +- generate merge, split, remainder, salvage, thinning, empty-transform, and + interactive-bypass queue operations and prove ledger conservation; +- split DEC mode 2031 subscribe and withdraw sequences across chunks, then + inject pending-cap salvage, rollback, transfer, reconnect, and source gaps; + prove fact order, exact scanner restoration/reset, no duplicate reply, and no + source settlement from projection metadata; +- put two clients at different display/source offsets in one shared span and + prove no resend, skip, or early reclamation; +- cover ASCII, BMP/CJK, surrogate pairs, unpaired surrogates, JSON-escaped + controls, and transformed spans where `rawLength !== data.length`; assert the + budget arithmetic and reject transformed frames without valid `rawLength`; +- run the shared semantic state-machine suite and the implemented SSH + socket/named-pipe adapter suite; local and daemon V1 adapters remain + unimplemented, direct-stdio/WSL has deterministic common-transport evidence + only, and remote-runtime source mapping is implemented at a deterministic + seam but unexecuted in a live paired topology; +- verify every new SSH session and reconnect offers V1 and every same-build + relay advertises and grants it; separately verify a capability-omitting old + client remains token-free and the narrow method-not-found relay fallback is + bounded; cover invalid/stale credential or lease rejection, 30-second lease + expiry, and atomic reconnect generation transfer through POSIX sockets and + Windows named pipes; prove constructor stdout, WSL child stdio, and an + unproved plain socket are never elected, and cover POSIX `0600` plus Windows + current-user ACLs; +- verify split shell/SFTP home discovery, per-launch marker creation under the + held install lock, credential write before lock release, canonical shell-path + fallback, system-SSH bypass, and unconfirmed teardown lock retention; +- verify token rotation on spawn, attach, reconnect, provider replacement, and + same-client duplicate attach/spawn; replacement must cancel the old token + once, report its exact remaining span, and transfer only matching recovery + coverage while canceling the already-checkpointed prefix without re-ingest, + duplicate output, or an open cursor; +- keep one lossless remote stream attached across an unsettled old-token frame + and a contiguous new-token frame; require strictly higher client and owner + generations, the same provider/incarnation, a new token, and reject every + stale or partially advanced identity; +- verify metadata response callbacks precede source-ranged recovery and live + data in V1, including saturation with older control; use multi-frame + all-control-character gaps to prove source bodies remain ordinary + producer-owned `pty.data` rather than control responses; +- feed response, recovery, completion fence, and first live data in one decoder + turn; prove synchronous `beforeResolve` installs a provisional cursor at + `checkpointSourceEndSu`; deliver completion before lease-held frames, then + prove validated reconnect transfers those frames only into private + quarantine, routes ranges through `recoveryEndSu`, and projects them once + only after exact body/fence validation. Pre-transfer rollback drops held + data, restores only the active predecessor, and awaits exact + token-cancellation proof; +- reject wrong-incarnation checkpoints and checkpoints below old credit or + beyond retained live end; require a token-free `restoreRequired` response + with no partial recovery or forward clamping; retain the canceled record + through response settlement, retire only the exact current record, then + prove retry mints a fresh token and publishes one live source frame; +- send identical recovery `pty.data` for distinct tokens within one second and + prove token/range identity, not a wall-clock fingerprint, controls admission; +- retry an operation-ID spawn after commit/before response from a new client; + prove it receives a fresh subscription and the stale client cannot ACK; +- assert `write(false)` admits exactly one frame, preserves the control reserve, + and admits no ordinary later frame before drain; +- queue ordinary PTY input before source ACK, cancellation, exit, request, and + response frames; after drain require liveness/control selection first, FIFO + within all three lanes, and one ordinary selection after four controls; +- deliver a cumulative ACK reentrantly before the source writer callback; + reserve only the exact same-token pending boundary, then prove successful + settlement applies it and failed settlement permits only an exact retry; +- inject asynchronous Node write-callback errors for session grants, token + activation, recovery completion, exit, and ACK publication; prove none + advance state and the client generation closes with exact cancellation; +- reject one `pty.recoveryComplete` admission, fire repeated capacity signals, + and prove one bounded retry is admitted; hold its callback and prove no live + output or duplicate completion passes before successful settlement; +- reject a PTY subscription whose empty non-reserved sink capacity is below + 8 KiB; prove every admitted transformed span remains writable; +- saturate relay fs/Git bulk; preserve fixed 256 KiB `fs.streamChunk` frames in + the exclusive empty-sink lane, slice only reshapable bulk/PTY data, and bound + cancellation, keepalive, ACK, immediate echo, and control latency; +- assert sentinel, handshake residue, and connect-mode socket data use one + stdout writer in FIFO order under saturation; +- assert close, error, detach, invalidate, replacement, reset, and dispose + settle/cancel each callback, token, and pause epoch exactly once; +- cover every lifecycle-table row and reject any + `received != open + transferring + settled + transferred + canceled` state; +- require `pty.deliveryCanceled` or generation-close proof for every + relay-initiated cancel and reject stale cancellation notifications; +- delay headless-emulator callbacks indefinitely; prove per-PTY/global model + queues and the one-frame intake slot plateau, upstream credit stops, low-water + resume is exact, and emulator rejection cancels rather than credits; +- while model admission is blocked, place cancellation/control behind a + bounded PTY burst; prove the 1 MiB/64-frame reserve services it without + reordering data, and prove reserve exhaustion pauses reads then closes the + provider at the deadline with cleanup proof; +- under the same pressure, inject `data -> pty.exit` and + `recovery pty.data -> pty.recoveryComplete`; prove both lifecycle fences + remain behind their token's quarantined data, while a same-token cancellation + proof bypasses only after atomically canceling that prefix; +- reject recovery cancellation publication and local proof application while + the attempt is current; prove the exact provider generation, mux, provider, + publishers, transferred activation, and registration close once, while + unrelated generations and physical PTY ownership remain untouched; +- prove renderer ACK/heal/write-off on V1 settles/transfers via display ranges + and emits no legacy `acknowledgeDataEvent` wire delta; +- admit interleaved spans from two token generations and prove every desktop + operation resolves the immutable span ID, PTY incarnation, delivery token, + source/display interval, and transform metadata; inject model-reservation, + projection-admission, send, merge/split, salvage, reload, and replacement + failure at each transaction boundary and prove rollback or exact transfer; +- generate remote UTF-8, JSON-escaped, transformed, split, and composite frames; + keep legacy `ackOutput: 1` delta ACKs outside source obligations; negotiate + `ackOutputSourceRanges: 1`, then apply partial, excessive, and stale-generation + cumulative ACKs and prove a partial ACK releases only its exact byte delta, + retains the covering source frame, and settles source only at recorded frame + boundaries, with send-stop without auto-transfer at the cap and atomic + mapping transfer on an explicit reconnect/detach; +- overflow ACK-pending remote output and inject snapshot serialization, + publication, exact-generation commit, and rollback failures; prove covered + chunks trim only after commit and recovery remains single-flight; after + cancellation proof reclaims a reserved span, require commit rejection, + idempotent rollback, and safe late settle/detach pruning; +- feed current mobile notification epochs and watermarks into terminal ACK + handlers and prove they are rejected without changing any source obligation; +- extend `terminal-output-frame-chunks-equivalence.test.ts` so composite source + identity preserves the current code-unit chunk text, encoded bytes, sequence + rounding, and allocation/performance contract; +- retain `terminal-stream-byte-length.test.ts` equivalence at its native-call + floor and over-limit boundary so transport byte accounting and batch flushes + cannot silently become source-unit accounting; +- hold desktop parse ACKs while a lossless mobile/web consumer advances; prove + desktop transfer to model restore allows upstream credit to continue; +- cover pause/resume hysteresis and re-entrant output/exit callbacks; +- drive `SshPtyProvider.pauseProducer`/`resumeProducer` through + `pty.setDeliveryPaused`; prove token/generation scoping, idempotency, native + pause when all required deliveries are ineligible, and continued progress + for a healthy additional subscriber; +- run one slow and one healthy client, verify only the slow client is detached; +- run no clients, verify no live pending data, bounded replay, and an unpaused + producer outside reconnect grace; +- verify exit follows data and never bypasses the window; after the exit write + callback, keep the sealed token and uncredited suffix live, accept a late + cumulative ACK, advance main `ackPublishedEndSu` only from its write callback, + and close the relay record only after it applies that ACK; separately prove + timeout publishes cancellation or uses generation-close proof and affects + only the stalled PTY token; +- receive exit with queued and in-flight emulator writes plus desktop and + remote mappings; prove main delays `runtime.onPtyExit`, renderer exit, model + disposal, and subscription cleanup until prior owners settle/transfer, and + prove deadline first acquires cancellation proof, transfers published + projections, commits proof, prepares/finalizes once, and rejects late + generation callbacks; close the generation while proof is pending and prove + no final exit publishes, then separately make proof fail while current and + prove only then the provider closes; +- close 2,048 sequential provider generations and prove one retained exact + closed range rejects early, middle, and latest stale events; close out of + order and prove an unclosed older generation remains admissible until its own + close merges the gap; +- verify round-robin progress across 50 continuously active PTYs; +- verify spans are freed only after all divergent cursors advance, cancel, or + transfer; +- test every source-unit and byte budget at limit minus one, limit, and plus + one, including global low-water hysteresis and legacy per-client/relay byte + publication caps plus their drain-owned low waters; +- verify decoder order, synchronous first frame, bounded yields, transport + pause/resume through ssh2, system SSH, WSL child stdio, stdin, Unix socket, + and named-pipe adapters, reset cancellation, partial 16 MiB frames, input-cap + close, oversized discard, and exact handshake residue; +- keep an unacked header across decoder/read self-pause and writer saturation; + prove resume rebases both health clocks, coalesces keepalives, caps timestamp + entries, and gives a full timeout window; +- feed hostile `ack=0xffffffff` and prove work is proportional to pending map + size, never the numeric ACK range; +- prove terminal obligations advance `obligationsTerminalEndSu` before any ACK + write, coalescing advances `ackQueuedEndSu`, callbacks alone advance + `ackPublishedEndSu`, and cleanup waits for the published end; cover failed + writes, cumulative retries, at most 64 entries, 8 ms/64 Ki-su flush rules, + and input fairness; +- fuzz frame boundaries, JSON sizes, reconnect timing, transport data during a + decoder continuation, and scalar-safe Unicode splits; +- exercise unconditional same-build V1 negotiation, old-client capability + omission, and method-not-found fallback; reject a manually mismatched + `.version`, and identify—but do not adopt or kill—an orphaned prior-version + daemon; +- verify the session grant rejects unrequested versions and zero, negative, + excessive, non-finite, or unsafe windows. + +Use a deterministic fake sink that records accepted frames separately from its +saturation signal. Use a fake native PTY whose `pause()` can synchronously emit +one last chunk to prove the transient overshoot bound. + +### Integration and E2E + +`tests/e2e/ssh-docker-relay-perf.spec.ts` now contains V1 evidence for the +implemented Linux SSH/deployed-relay path. The test harness exercises the +normal deployment and connection path with no override and: + +1. stalls desktop ACK and observes an exact 256 Ki-source-unit negotiated + plateau while a second SSH PTY remains responsive; +2. preserves fixed `STREAM_CHUNK_SIZE` filesystem semantics and completes Git + churn without corruption while active typing stays within budget; +3. reconnects the negotiated owner lease and proves the existing SSH + workspace terminal remains usable; +4. retains the direct typing latency control. + +Writer `write(false)`/drain, callback failure, liveness bypass, decoder +scheduling, sealed exit, stale ACK, exact recovery, and remote replacement are +proved at deterministic unit/service seams. The Docker suite does not force +the main-to-relay sink to remain saturated beyond the health deadline, does +not prove paired-runtime behavior, and must not be cited for those claims. + +Docker SSH proves the Linux SSH provider/relay path only. It must never be +reported as headed paired-server, headless `orca serve`, remote-runtime +lossless subscription, macOS node-pty, Windows ConPTY/named pipe, or WSL +evidence. + +The Slice 5d mapping is implemented, but live paired-runtime coverage remains +a separate promotion requirement: + +| Topology | Required oracle | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------- | +| Headed paired desktop server + separate client | host-owned PTY identity, source mapping, client ACK/restore, reconnect, and cleanup | +| Headless `orca serve` + same client flow | identical source settlement plus headless model admission/startup ownership | +| Direct Docker SSH | deployed relay transport, main desktop intake, real remote process, writable/drain behavior | +| WSL child stdio | sentinel/residue, read pause, writer drain, no POSIX-path assumptions | +| Windows native | ConPTY pause/resume and named-pipe callbacks/drain | +| Local daemon | authenticated paired hello and exact-version legacy fallback | +| Local provider | direct semantic instantiation with no relay framing | + +Each topology also runs in a folder workspace fixture with no `.git`; source +identity and cleanup never depend on worktree metadata. Prior-version daemons +remain version-scoped legacy processes: report them, do not negotiate V1, +adopt their PTYs, or claim the new main bounds their memory. Keep any new +native artifact compatible with Ubuntu 20.04/glibc 2.31; this design itself +adds none. + +The +`terminal-performance.output-backpressure-budget` reliability gate remains +`experimental` and `partial`. Its `coveredProviders: ["ssh"]` means the +deterministic direct-SSH contracts plus the macOS-hosted Docker +OpenSSH/deployed-relay run described above. It does not cover every SSH host +platform and does not mark local, daemon, remote-runtime, headed, headless, +WSL, Windows, folder-workspace, prior-version, mixed-version, or Ubuntu 20.04 +packaging topologies as executed. The WSL stdin write/callback/drain contract +is deterministic adapter evidence only, not a physical WSL run. + +## Compatibility and release validation + +Every new SSH session and automatic reconnect offers V1. Every relay from the +same build advertises and grants V1, so the normal deployment path always uses +source credit. Sink drain, the reserved writer lane, stdout serialization, +bounded decoder work, bulk admission, and hostile header-ACK hardening are +also unconditional. + +Compatibility behavior is explicit: + +1. An old client that omits the V1 capability remains token-free and uses the + bounded legacy writer. +2. A new main may use bounded legacy delivery only when the same-build + compatibility check accepts JSON-RPC method-not-found for + `pty.openClient`. +3. A successful `pty.openClient` response without the offered V1 grant, an + invalid grant, or another request error fails closed rather than silently + downgrading. +4. Disconnect performs normal token-scoped cancellation and exit cleanup; it + never rotates a live V1 token into legacy mode in place. +5. A prior-version orphan remains governed by its own version-scoped binary and + cleanup. The new main reports but does not adopt, kill, or claim to bound it. + +Implementation and validation status: + +1. Slices 1-4 are implemented under legacy-compatible framing with narrow + reliability oracles. +2. Slices 5a-5c implement unconditional V1 offer/advertisement, source credit, + exit, and reconnect with unit/service contracts and direct Docker SSH + evidence. +3. Slices 5d-5e implement remote source-range transactions and the final + lifecycle reconciliation at deterministic main/runtime/provider/relay + seams. Live headed paired, headless, WSL, Windows, local-daemon, + local-provider, and folder-workspace evidence remains future work before + those topologies are marked covered. +4. Current-working-tree deterministic reconciliation and focused integrated-fix checks + are complete. The isolated reconnect and final four-case direct-SSH Docker + runs, full repository typecheck, and full lint/reliability suite are green; + final exact-implementation-head architecture and transport reviews found no + code blocker, and the transport review's stale-provenance finding is closed + by the exact artifact record above. + +Release criteria: + +- no unbounded increase across 30-minute desktop-only, slow-model, and required + lossless-consumer floods; +- relay and main stay within the documented budgets plus 25% allocator slack; +- active echo p95 remains below 100 ms under a background flood; +- exact logical output sequence after resume; +- no leaked obligations, writer callbacks, drain waiters, cursors, tokens, + decoder tasks, or paused PTYs; +- no false dead-link reconnect across a 30-minute saturated/self-paused run; +- bounded time-to-last-reattach with one injected per-PTY failure; +- unconditional same-build V1 negotiation, bounded legacy compatibility, + manual mismatch rejection, orphan reporting, and Linux/macOS/Windows/WSL + smoke tests pass. + +## Calibrated findings and non-goals + +- The liveness risk is real for transport saturation and self-paused decoding. + A renderer-only stall is absorbed by bounded projection transfer/restore + while required-model credit continues. A stalled model admission or + negotiated lossless remote consumer exhausts its window and pauses the + native PTY; tests keep these distinct. +- V1 uses logical priority lanes and reserved writable capacity, not a second + SSH channel. Adding a physical control channel would widen handshake, + reconnect, system-SSH, and legacy compatibility scope without being required + once PTY bursts and decoder pauses are bounded. +- There is no arbitrary 16-socket cap. Additional PTY projections are bounded + and isolated by their own writer/publication capacity; short-lived remote CLI + clients remain independent. +- A 60-second watchdog may warn about an old open obligation, but it cannot + force-credit data. Recovery must use a proven transfer or token cancellation. +- Reconnect-grace data is a bounded live-delivery owner, not an expansion of the + 100 Ki replay-tail contract. Expiry is an explicit data gap, never silent + truncation. +- Content-hashed endpoints make mixed production binaries unreachable. The + real upgrade skew is an orphaned prior-version daemon outside the new main's + startup-gated endpoint; V1 reports but cannot adopt, cancel, or retroactively + bound it. +- Preserving the real exit code for a PTY that exits before any new client + reattaches would require attach-visible tombstone retention. V1 documents and + tests the existing synthetic `-1` fallback instead of coupling that separate + product change to backpressure. + +## Reference-commit assessment + +Retain from `1500a92904` the useful concepts of generation tokens, client +isolation, pause/resume, targeted dispatcher sends, cleanup of stale credits, +and slow-client eviction. + +Do not reproduce these behaviors: + +- ordinary notifications continuing to write after `write(false)`; +- delta ACKs without cumulative token scope; +- automatic live broadcast to every connected client; +- force-flushing final PTY output beyond the credit window; +- per-client windows without aggregate retained-memory budgets; +- synchronous all-frame draining; +- assuming `256 Ki × 50 PTYs × N clients` is itself a safe memory bound. + +This design makes writable-drain state, application credit, retained memory, +and decoder CPU four explicit and independently enforced bounds. + +## Platform and workspace requirements + +The relay can run on a native macOS, Linux, Windows, or WSL host reached over +SSH. Use Node stream APIs and runtime platform checks; do not assume POSIX file +descriptors, path separators, signals, or Unix sockets. Windows uses ConPTY and +named pipes, while POSIX uses node-pty and Unix sockets, but both must expose +the same idempotent pause/resume and sink state contract. + +Network latency changes ACK cadence but not the window invariant. A +high-bandwidth/high-latency SSH link may need later window tuning based on +telemetry; it must not receive an unbounded adaptive window. SSH disconnects +are normal lifecycle events, not exceptional shortcuts around cleanup. + +No part of subscription identity or cleanup may depend on `.git`, a worktree +ID, or a Git provider. Folder workspaces use the same connection, PTY, and +client generations as Git worktrees. Bulk Git changes only slice and schedule +existing response frames; they add no Git command, provider-specific behavior, +or requirement beyond the Git 2.25 core-workflow baseline. diff --git a/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts b/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts index a8085485553..b57be2544b6 100644 --- a/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts +++ b/src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts @@ -16,14 +16,20 @@ type FakeChild = ChildProcessWithoutNullStreams & { kill: ReturnType + resume: ReturnType + } stderr: EventEmitter - stdin: { write: ReturnType } + stdin: EventEmitter & { write: ReturnType } kill: ReturnType } - child.stdout = new EventEmitter() + child.stdout = Object.assign(new EventEmitter(), { + pause: vi.fn(), + resume: vi.fn() + }) child.stderr = new EventEmitter() - child.stdin = { write: vi.fn(() => true) } + child.stdin = Object.assign(new EventEmitter(), { write: vi.fn(() => true) }) child.kill = vi.fn() return child as unknown as FakeChild } @@ -62,6 +68,10 @@ describe('waitForWslRelaySentinel', () => { const transport = await promise expect(typeof transport.write).toBe('function') expect(typeof transport.onData).toBe('function') + transport.pauseReads?.() + transport.resumeReads?.() + expect(child.stdout.pause).toHaveBeenCalledOnce() + expect(child.stdout.resume).toHaveBeenCalledOnce() }) it('resolves past leading garbage and hands trailing bytes to onData', async () => { @@ -149,4 +159,25 @@ describe('waitForWslRelaySentinel', () => { expect(err.message).toContain('E_FAIL') expect(err.message).not.toContain(String.fromCharCode(0)) }) + + it('forwards write(false), callback settlement, and drain from WSL stdin', async () => { + const child = fakeChild() + const callback = vi.fn() + const drain = vi.fn() + const writeMock = child.stdin.write as ReturnType + writeMock.mockImplementation((_data, onWritten) => { + onWritten(null) + return false + }) + const promise = waitForWslRelaySentinel(child) + emitStdout(child, RELAY_SENTINEL) + const transport = await promise + + transport.onDrain?.(drain) + expect(transport.write(Buffer.from('frame'), callback)).toBe(false) + expect(callback).toHaveBeenCalledWith({ ok: true }) + child.stdin.emit('drain') + expect(drain).toHaveBeenCalledOnce() + expect(transport.supportsWriteSettlement).toBe(true) + }) }) diff --git a/src/main/agent-hooks/wsl-hook-relay-sentinel.ts b/src/main/agent-hooks/wsl-hook-relay-sentinel.ts index fdabf70fe15..5e1b5b5a872 100644 --- a/src/main/agent-hooks/wsl-hook-relay-sentinel.ts +++ b/src/main/agent-hooks/wsl-hook-relay-sentinel.ts @@ -123,12 +123,15 @@ export function waitForWslRelaySentinel( pendingChunks.push(trailing) } const transport: MultiplexerTransport = { - write: (data) => { - try { - child.stdin.write(data) - } catch { - // Channel already closing — mux close handling takes over. - } + write: (data, onSettled) => { + return child.stdin.write(data, (error?: Error | null) => { + onSettled?.(error ? { ok: false, error } : { ok: true }) + }) + }, + supportsWriteSettlement: true, + onDrain: (cb) => { + child.stdin.on('drain', cb) + return () => child.stdin.off('drain', cb) }, onData: (cb) => { dataCallbacks.push(cb) @@ -143,6 +146,8 @@ export function waitForWslRelaySentinel( } }, onClose: (cb) => closeCallbacks.push(cb), + pauseReads: () => child.stdout.pause(), + resumeReads: () => child.stdout.resume(), close: () => child.kill() } resolve(transport) diff --git a/src/main/ipc/pty-pending-data-drain-contract.ts b/src/main/ipc/pty-pending-data-drain-contract.ts index 05fbd2ff77e..d1d4c5be1a8 100644 --- a/src/main/ipc/pty-pending-data-drain-contract.ts +++ b/src/main/ipc/pty-pending-data-drain-contract.ts @@ -9,6 +9,8 @@ export type PendingPtyData = { droppedOutput?: true droppedMode2031Data?: string droppedMode2031ScanState?: Mode2031ReplyScanState + projectionAdmissionIds?: readonly string[] + projectionAdmissionsTransferred?: true } export type PtyPendingDataDrainDisposition = 'active' | 'background' | 'blocked' diff --git a/src/main/ipc/pty-pending-projection-admissions.test.ts b/src/main/ipc/pty-pending-projection-admissions.test.ts new file mode 100644 index 00000000000..62863e27fa6 --- /dev/null +++ b/src/main/ipc/pty-pending-projection-admissions.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it, vi } from 'vitest' +import { + appendPendingProjectionAdmission, + compactPendingProjectionAdmissions, + PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS, + propagatePendingProjectionRemainder +} from './pty-pending-projection-admissions' + +describe('pending PTY projection admissions', () => { + it('retains the exact cap then transfers the whole ordered run on overflow', () => { + const transfer = vi.fn() + let state = compactPendingProjectionAdmissions() + + for (let index = 0; index < PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS; index++) { + state = appendPendingProjectionAdmission(state, `projection-${index}`, { + isPending: () => true, + transfer + }) + } + + expect(state.projectionAdmissionIds).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS) + expect(transfer).not.toHaveBeenCalled() + + state = appendPendingProjectionAdmission(state, 'projection-overflow', { + isPending: () => true, + transfer + }) + + expect(state).toEqual({ projectionAdmissionsTransferred: true }) + expect(transfer).toHaveBeenCalledOnce() + expect(transfer.mock.calls[0]?.[0]).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS + 1) + }) + + it('compacts terminal prefixes and transfers later admissions until the remainder drains', () => { + const transfer = vi.fn() + const compacted = propagatePendingProjectionRemainder( + { + projectionAdmissionIds: ['projection-published', 'projection-partial', 'projection-tail'] + }, + { sent: true, projectionsTransferred: false }, + { isPending: (id) => id !== 'projection-published', transfer } + ) + + expect(compacted).toEqual({ + projectionAdmissionIds: ['projection-partial', 'projection-tail'] + }) + + const transferred = appendPendingProjectionAdmission( + { projectionAdmissionsTransferred: true }, + 'projection-after-transfer', + { isPending: () => true, transfer } + ) + expect(transferred).toEqual({ projectionAdmissionsTransferred: true }) + expect(transfer).toHaveBeenCalledWith(['projection-after-transfer'], 'pending-projection-cap') + + expect( + propagatePendingProjectionRemainder( + compacted, + { sent: true, projectionsTransferred: true }, + { isPending: () => true, transfer } + ) + ).toEqual({ projectionAdmissionsTransferred: true }) + }) +}) diff --git a/src/main/ipc/pty-pending-projection-admissions.ts b/src/main/ipc/pty-pending-projection-admissions.ts new file mode 100644 index 00000000000..9fa0d423bc9 --- /dev/null +++ b/src/main/ipc/pty-pending-projection-admissions.ts @@ -0,0 +1,57 @@ +export const PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS = 1024 + +export type PendingProjectionAdmissions = Readonly<{ + projectionAdmissionIds?: readonly string[] + projectionAdmissionsTransferred?: true +}> + +type PendingProjectionAdmissionOptions = Readonly<{ + isPending: (id: string) => boolean + transfer: (ids: readonly string[], reason: string) => void +}> + +const DEFAULT_OPTIONS: PendingProjectionAdmissionOptions = { + isPending: () => true, + transfer: () => {} +} + +export function compactPendingProjectionAdmissions( + state: PendingProjectionAdmissions = {}, + options: PendingProjectionAdmissionOptions = DEFAULT_OPTIONS +): PendingProjectionAdmissions { + if (state.projectionAdmissionsTransferred) { + return { projectionAdmissionsTransferred: true } + } + const ids = Array.from( + new Set((state.projectionAdmissionIds ?? []).filter((id) => options.isPending(id))) + ) + return ids.length > 0 ? { projectionAdmissionIds: ids } : {} +} + +export function appendPendingProjectionAdmission( + state: PendingProjectionAdmissions, + id: string, + options: PendingProjectionAdmissionOptions +): PendingProjectionAdmissions { + if (state.projectionAdmissionsTransferred) { + options.transfer([id], 'pending-projection-cap') + return { projectionAdmissionsTransferred: true } + } + const compacted = compactPendingProjectionAdmissions(state, options) + const ids = [...(compacted.projectionAdmissionIds ?? []), id] + if (ids.length <= PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS) { + return { projectionAdmissionIds: ids } + } + options.transfer(ids, 'pending-projection-cap') + return { projectionAdmissionsTransferred: true } +} + +export function propagatePendingProjectionRemainder( + state: PendingProjectionAdmissions, + delivery: Readonly<{ sent: boolean; projectionsTransferred: boolean }>, + options: PendingProjectionAdmissionOptions +): PendingProjectionAdmissions { + return delivery.sent && !delivery.projectionsTransferred + ? compactPendingProjectionAdmissions(state, options) + : { projectionAdmissionsTransferred: true } +} diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index b04550084cc..59b69ea25d0 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -247,6 +247,11 @@ import { resolveWindowsShellLaunchArgs } from '../providers/windows-shell-args' import { _resetWslCachesForTests, _setWslCachesForTests } from '../wsl' import { wslHookRelayManager } from '../agent-hooks/wsl-hook-relay-manager' import { acquireWatcherRemovalGate } from './watcher-removal-gate' +import { + acceptSshPtyOutputData, + acceptSshPtyOutputExit, + closeSshPtyOutputGeneration +} from './ssh-pty-output-intake-registry' // Why: Windows resolves a bare PowerShell name to an absolute exe before ConPTY, else CreateProcessW fails with error 5 (PR #6537 / #5161). const RESOLVED_WINDOWS_POWERSHELL = 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' @@ -449,7 +454,8 @@ describe('registerPtyHandlers', () => { 'ssh-reattach-1', 'ssh-reattach-fail', 'ssh-reattach-ok', - 'ssh-runtime-env' + 'ssh-runtime-env', + 'ssh-generation-replacement' ]) { unregisterSshPtyProvider(leakedConnectionId) } @@ -11556,6 +11562,197 @@ describe('registerPtyHandlers', () => { } }) + it('keeps negotiated source-credit overflow off the legacy PTY-global pause path', async () => { + vi.useFakeTimers() + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + const provider = installObservableDaemonTestProvider() + let modelSequence = 0 + const runtime = { + setPtyController: vi.fn(), + setRemoteTerminalSourceRangeConsumerHooks: vi.fn(), + getPtyOutputSequence: vi.fn(() => modelSequence), + onPtyData: vi.fn( + (_id: string, data: string, _at: number, rawLength = data.length) => + (modelSequence += rawLength) + ), + acceptPtyDataBounded: vi.fn( + (_id: string, _data: string, _at: number, rawLength: number) => { + modelSequence += rawLength + return { sequence: modelSequence, completion: Promise.resolve() } + } + ) + } + registerPtyHandlers(mainWindow as never, runtime as never) + mainWindow.webContents.send.mockClear() + + const sourceChunk = 's'.repeat(128 * 1024) + for (let index = 0; index < 17; index++) { + const sourceStartSu = index * sourceChunk.length + await acceptSshPtyOutputData({ + id: 'source-credit-pty', + data: sourceChunk, + providerGeneration: 41, + ptyIncarnation: 'source-incarnation', + rawLength: sourceChunk.length, + transformed: false, + source: { + relayPtyId: 'relay-source-pty', + spanId: `source-token:${sourceStartSu}:${sourceStartSu + sourceChunk.length}`, + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'source-token', + sourceStartSu, + sourceEndSu: sourceStartSu + sourceChunk.length + } + }) + } + + expect(getPtyRendererDeliveryDebugSnapshot()).toMatchObject({ + pendingPtyCount: 1, + pendingChars: 0 + }) + expect(provider.pauseProducer).not.toHaveBeenCalledWith('source-credit-pty') + expect(provider.resumeProducer).not.toHaveBeenCalledWith('source-credit-pty') + + provider.emitData('legacy-pty', 'l'.repeat(320 * 1024)) + expect(provider.pauseProducer).toHaveBeenCalledTimes(1) + expect(provider.pauseProducer).toHaveBeenCalledWith('legacy-pty') + expect(provider.pauseProducer).not.toHaveBeenCalledWith('unrelated-pty') + + vi.runAllTimers() + expect(provider.resumeProducer).toHaveBeenCalledTimes(1) + expect(provider.resumeProducer).toHaveBeenCalledWith('legacy-pty') + } finally { + errorSpy.mockRestore() + vi.useRealTimers() + } + }) + + it('pauses and resumes the exact SSH provider generation across reconnect replacement', async () => { + vi.useFakeTimers() + const completion = makeDeferred() + let sequence = 0 + let captures = 0 + const runtime = { + setPtyController: vi.fn(), + setRemoteTerminalSourceRangeConsumerHooks: vi.fn(), + getPtyOutputSequence: vi.fn(() => sequence), + acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => { + sequence += rawLength + captures++ + return { + sequence, + completion: captures === 1 ? completion.promise : Promise.resolve() + } + }) + } + const original = { + providerGeneration: 41, + hasPtyDeliveryPauseAdapter: () => true, + pauseProducer: vi.fn(), + resumeProducer: vi.fn() + } + const replacement = { + providerGeneration: 42, + hasPtyDeliveryPauseAdapter: () => true, + pauseProducer: vi.fn(), + resumeProducer: vi.fn() + } + const id = 'ssh:ssh-generation-replacement@@relay-pty' + const receipts: Promise[] = [] + + try { + registerPtyHandlers(mainWindow as never, runtime as never) + registerSshPtyProvider('ssh-generation-replacement', original as never) + const running = acceptSshPtyOutputData({ + id, + data: 'a'.repeat(256 * 1024), + providerGeneration: 41, + ptyIncarnation: 'incarnation-41', + rawLength: 256 * 1024, + transformed: false + }) + receipts.push(running) + registerSshPtyProvider('ssh-generation-replacement', replacement as never) + const pressured = acceptSshPtyOutputData({ + id, + data: 'b', + providerGeneration: 41, + ptyIncarnation: 'incarnation-41', + rawLength: 1, + transformed: false + }) + receipts.push(pressured) + + expect(original.pauseProducer).toHaveBeenCalledWith(id) + expect(replacement.pauseProducer).not.toHaveBeenCalled() + + completion.resolve() + await Promise.all([running, pressured]) + expect(original.resumeProducer).toHaveBeenCalledWith(id) + expect(replacement.resumeProducer).not.toHaveBeenCalled() + } finally { + completion.resolve() + await Promise.allSettled(receipts) + closeSshPtyOutputGeneration(41, 'test-cleanup') + unregisterSshPtyProvider('ssh-generation-replacement') + } + }) + + it('rejects local data while an SSH renderer exit waits for projection settlement', async () => { + const provider = installObservableDaemonTestProvider() + let sequence = 0 + const runtime = { + setPtyController: vi.fn(), + setRemoteTerminalSourceRangeConsumerHooks: vi.fn(), + getPtyOutputSequence: vi.fn(() => sequence), + acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => { + sequence += rawLength + return { sequence, completion: Promise.resolve() } + }), + onPtyData: vi.fn(), + onPtyExit: vi.fn() + } + const id = 'ssh:exit-data-race@@relay-pty' + + registerPtyHandlers(mainWindow as never, runtime as never) + mainWindow.webContents.send.mockClear() + await acceptSshPtyOutputData({ + id, + data: 'before-exit', + providerGeneration: 51, + ptyIncarnation: 'incarnation-51', + rawLength: 'before-exit'.length, + transformed: false + }) + const exit = acceptSshPtyOutputExit({ + id, + code: 0, + providerGeneration: 51, + ptyIncarnation: 'incarnation-51' + }) + await Promise.resolve() + + provider.emitData(id, 'must-not-follow-exit') + expect(mainWindow.webContents.send).not.toHaveBeenCalledWith('pty:data', { + id, + data: 'must-not-follow-exit' + }) + + getPtyAckDataListener()(null, { id, processedChars: 'before-exit'.length }) + await exit + expect(mainWindow.webContents.send.mock.calls.at(-1)).toEqual([ + 'pty:exit', + { + id, + code: 0, + providerGeneration: 51, + ptyIncarnation: 'incarnation-51' + } + ]) + }) + it('resumes a paused producer when the PTY exits before draining', async () => { vi.useFakeTimers() try { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 3d3c4811747..ad0b68cdaa2 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -158,6 +158,19 @@ import { unmarkHiddenRendererPty } from './pty-hidden-delivery-gate' import { PtyPendingDataDrainQueue, type PendingPtyData } from './pty-pending-data-drain-queue' +import { + appendPendingProjectionAdmission, + compactPendingProjectionAdmissions, + propagatePendingProjectionRemainder, + type PendingProjectionAdmissions +} from './pty-pending-projection-admissions' +import { SshPtyOutputIntake } from './ssh-pty-output-intake' +import { + cancelSshPtySourceDelivery, + installSshPtyOutputIntake, + publishSshPtySourceAck +} from './ssh-pty-output-intake-registry' +import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' import { clearNativeWindowsConptyPty, isNativeWindowsLocalPtySpawn, @@ -202,6 +215,7 @@ type FreshLocalFallbackProvider = IPtyProvider & { routesFreshSpawnsToLocalProvider?: true } const sshProviders = new Map() +const sshProvidersByGeneration = new Map() type RegisteredPtyProvider = { provider: IPtyProvider @@ -1296,10 +1310,19 @@ function beginPtySpawnForWorktree( /** Register an SSH PTY provider for a connection. */ export function registerSshPtyProvider(connectionId: string, provider: IPtyProvider): void { sshProviders.set(connectionId, provider) + const generation = (provider as { providerGeneration?: number }).providerGeneration + if (Number.isSafeInteger(generation) && generation! > 0) { + sshProvidersByGeneration.set(generation!, provider) + } } /** Remove an SSH PTY provider when a connection is closed. */ export function unregisterSshPtyProvider(connectionId: string): void { + const provider = sshProviders.get(connectionId) + const generation = (provider as { providerGeneration?: number } | undefined)?.providerGeneration + if (generation !== undefined && sshProvidersByGeneration.get(generation) === provider) { + sshProvidersByGeneration.delete(generation) + } sshProviders.delete(connectionId) } @@ -1467,6 +1490,7 @@ let rendererDidStartLoadingHandler: (() => void) | null = null // Why: Restart daemon must re-bind provider→renderer listeners after replaceDaemonProvider swaps localProvider, else subscribers stay bound to the disposed adapter and new PTY data silently drops. let rebindProviderListeners: (() => void) | null = null +let sshOutputIntakeCleanup: (() => void) | null = null export function rebindLocalProviderListeners(): void { rebindProviderListeners?.() @@ -1840,8 +1864,10 @@ export function registerPtyHandlers( }, () => isHiddenPtyDeliveryGateEnabled(getSettings?.()) ) + let sshOutputIntake: SshPtyOutputIntake | null = null // Why: resuming a paused producer during exit can synchronously emit; those bytes must not queue behind pty:exit. const rendererExitingPtyIds = new Set() + const rendererCreditBeforeExitByPty = new Map() const rendererDeliveryRestoreNeededPtys = new Set() function transitionHiddenRendererPtyDeliveryState(id: string, hidden: boolean) { @@ -1927,11 +1953,18 @@ export function registerPtyHandlers( pauseProducer: (id) => tryGetProviderForPty(id)?.pauseProducer?.(id), resumeProducer: (id) => tryGetProviderForPty(id)?.resumeProducer?.(id) }) + const sourceCreditPendingPtys = new Set() function updateProducerFlowControl(id: string): void { if (!PRODUCER_FLOW_CONTROL_ENABLED) { return } + if (sourceCreditPendingPtys.has(id)) { + if (pendingData.get(id)) { + return + } + sourceCreditPendingPtys.delete(id) + } producerFlowControl.update(id, pendingData.get(id)?.data.length ?? 0) } @@ -2000,7 +2033,16 @@ export function registerPtyHandlers( } function clearPendingPtyData(): void { + for (const pending of pendingData.values()) { + if (pending.projectionAdmissionIds) { + sshOutputIntake?.transferProjections( + pending.projectionAdmissionIds, + 'renderer-lifecycle-reset' + ) + } + } pendingData.clear() + sourceCreditPendingPtys.clear() } function readCurrentPtyRendererDeliveryDebugSnapshot(): PtyRendererDeliveryDebugSnapshot { @@ -2169,6 +2211,9 @@ export function registerPtyHandlers( producerFlowControl.releaseAll() clearDeliveryResyncProbe() deliveryResyncUnansweredWarnLogged = false + for (const id of rendererDeliveryAccountingByPty.keys()) { + sshOutputIntake?.transferPtyProjections(id, 'renderer-lifecycle-reset') + } rendererDeliveryAccountingByPty.clear() rendererInFlightTotalChars = 0 clearPendingPtyData() @@ -2265,6 +2310,9 @@ export function registerPtyHandlers( accounting.lastAckAtMs = Date.now() } rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - acknowledged) + if (acknowledged > 0) { + sshOutputIntake?.settleProjectionPrefix(id, acknowledged) + } return acknowledged } @@ -2336,6 +2384,12 @@ export function registerPtyHandlers( // Why drop pending: everything at/before markerSeq comes from the snapshot, so flushing pre-marker bytes would double-paint the restore. const pending = pendingData.get(id) if (pending) { + if (pending.projectionAdmissionIds) { + sshOutputIntake?.transferProjections( + pending.projectionAdmissionIds, + 'renderer-delivery-writeoff' + ) + } pendingDroppedChars += pending.data.length deletePendingPtyData(id) pendingOverflowMarkedPtys.delete(id) @@ -2365,7 +2419,11 @@ export function registerPtyHandlers( return writtenOff } - function sendPtyDataToRenderer(id: string, payload: PtyDataPayload): boolean { + function sendPtyDataToRenderer( + id: string, + payload: PtyDataPayload, + projectionAdmissionIds?: readonly string[] + ): { sent: boolean; projectionsTransferred: boolean } { const charCount = getPtyPayloadCharCount(payload) const accounting = rendererDeliveryAccountingByPty.get(id) const hadAccounting = accounting !== undefined @@ -2400,12 +2458,28 @@ export function registerPtyHandlers( } } rendererDeliveryRestoreNeededPtys.add(id) + if (projectionAdmissionIds) { + sshOutputIntake?.transferProjections(projectionAdmissionIds, 'renderer-send-failed') + } mainDeliveryBreadcrumbs.record('pty-data-send-failed', { id: redactPtyIdForDiagnostics(id), chars: charCount }) console.error('[pty] renderer data send failed; payload will not be retried', error) - return false + return { sent: false, projectionsTransferred: projectionAdmissionIds !== undefined } + } + let projectionsTransferred = false + if (projectionAdmissionIds) { + try { + sshOutputIntake?.publishProjectionPrefix( + projectionAdmissionIds, + payload.data.length, + charCount + ) + } catch { + sshOutputIntake?.transferProjections(projectionAdmissionIds, 'projection-publish-failed') + projectionsTransferred = true + } } if (rendererDeliveryRestoreNeededPtys.has(id)) { try { @@ -2418,7 +2492,7 @@ export function registerPtyHandlers( ) } } - return true + return { sent: true, projectionsTransferred } } function rendererPtyIsKnownHidden(id: string): boolean { @@ -2521,6 +2595,9 @@ export function registerPtyHandlers( pendingOverflowMarkedPtys.add(id) } pendingDroppedChars += pending.data.length + if (pending.projectionAdmissionIds) { + sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'pending-cap') + } const mode2031 = scanDroppedMode2031Data(pending.data, INITIAL_MODE_2031_REPLY_SCAN_STATE) // Why no trimmed content tail: a mid-stream gap would corrupt the pane; the droppedOutput sentinel repaints from the snapshot and realigns by sequence (only query bytes ride along). return { @@ -2531,6 +2608,39 @@ export function registerPtyHandlers( } } + function updatePendingProjectionAdmissions( + pending: PendingPtyData, + state: PendingProjectionAdmissions + ): void { + delete pending.projectionAdmissionIds + delete pending.projectionAdmissionsTransferred + if (state.projectionAdmissionIds) { + pending.projectionAdmissionIds = state.projectionAdmissionIds + } + if (state.projectionAdmissionsTransferred) { + pending.projectionAdmissionsTransferred = true + } + } + + function compactPendingProjectionState( + pending: PendingProjectionAdmissions, + projectionSemanticsId?: string + ): PendingProjectionAdmissions { + const options = pendingProjectionAdmissionOptions() + const compacted = compactPendingProjectionAdmissions(pending, options) + return projectionSemanticsId + ? appendPendingProjectionAdmission(compacted, projectionSemanticsId, options) + : compacted + } + + function pendingProjectionAdmissionOptions() { + return { + isPending: (id: string) => sshOutputIntake?.hasUnpublishedProjection(id) ?? false, + transfer: (ids: readonly string[], reason: string) => + sshOutputIntake?.transferProjections(ids, reason) + } + } + function appendPendingPtyData( id: string, existing: PendingPtyData | undefined, @@ -2539,10 +2649,14 @@ export function registerPtyHandlers( preservesSeq: boolean, containsBackgroundOutput: boolean, rawLength = data.length, - transformed = false + transformed = false, + projectionSemanticsId?: string ): PendingPtyData { // Why stay dropped at O(1): once over the cap the restore sentinel supersedes interim bytes; queries still get carved out (bounded) so replies survive the whole episode. if (existing?.droppedOutput === true) { + if (projectionSemanticsId) { + sshOutputIntake?.transferProjections([projectionSemanticsId], 'pending-cap') + } const mode2031 = scanDroppedMode2031Data( data, existing.droppedMode2031ScanState ?? INITIAL_MODE_2031_REPLY_SCAN_STATE @@ -2559,16 +2673,19 @@ export function registerPtyHandlers( droppedMode2031ScanState: mode2031.state } } + const projectionState = compactPendingProjectionState(existing ?? {}, projectionSemanticsId) const nextContainsBackgroundOutput = existing?.containsBackgroundOutput === true || containsBackgroundOutput if (!existing) { - return dropOversizedPendingPtyData(id, { + const pending: PendingPtyData = { data, ...(typeof startSeq === 'number' ? { startSeq } : {}), ...(rawLength !== data.length ? { rawLength } : {}), ...(transformed ? { transformed: true } : {}), ...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {}) - }) + } + updatePendingProjectionAdmissions(pending, projectionState) + return dropOversizedPendingPtyData(id, pending) } const existingRawLength = existing.rawLength ?? existing.data.length const next: PendingPtyData = { @@ -2578,6 +2695,7 @@ export function registerPtyHandlers( : {}), ...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {}) } + updatePendingProjectionAdmissions(next, projectionState) if (typeof existing.startSeq === 'number') { next.startSeq = existing.startSeq } @@ -2661,6 +2779,9 @@ export function registerPtyHandlers( pendingOverflowMarkedPtys.delete(id) updateProducerFlowControl(id) const drop = recordHiddenRendererPtyDataDrop(id, pending.data.length) + if (pending.projectionAdmissionIds) { + sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop') + } warnIfDroppingHiddenBytesForVisiblePty(id, pending.data.length) if (drop.shouldEmitRestoreMarker) { sendModelRestoreNeededMarker(id, 'hidden-drop', runtime?.getPtyOutputSequence(id)) @@ -2676,11 +2797,15 @@ export function registerPtyHandlers( updateProducerFlowControl(id) // Why droppedOutput sentinel: pending-cap drop means the pane must repaint from the snapshot, not continue a gapped stream (data = carved query bytes only). if ( - !sendPtyDataToRenderer(id, { + !sendPtyDataToRenderer( id, - data: pending.data + getDroppedMode2031RendererData(pending), - droppedOutput: true - }) + { + id, + data: pending.data + getDroppedMode2031RendererData(pending), + droppedOutput: true + }, + pending.projectionAdmissionIds + ).sent ) { sendFailed = true break @@ -2692,33 +2817,50 @@ export function registerPtyHandlers( const indivisible = pending.transformed === true const chunk = indivisible ? data : data.slice(0, PTY_BATCH_FLUSH_CHUNK_CHARS) const remaining = indivisible ? '' : data.slice(PTY_BATCH_FLUSH_CHUNK_CHARS) + let nextPending: PendingPtyData | undefined if (remaining) { - const nextPending: PendingPtyData = { data: remaining } + nextPending = { data: remaining } if (typeof pending.startSeq === 'number') { nextPending.startSeq = pending.startSeq + chunk.length } if (pending.containsBackgroundOutput === true) { nextPending.containsBackgroundOutput = true } + if (pending.projectionAdmissionIds) { + nextPending.projectionAdmissionIds = pending.projectionAdmissionIds + } + if (pending.projectionAdmissionsTransferred) { + nextPending.projectionAdmissionsTransferred = true + } pendingData.replaceWithRemainder(selection, nextPending) } else { pendingData.remove(selection) pendingOverflowMarkedPtys.delete(id) } updateProducerFlowControl(id) - if ( - !sendPtyDataToRenderer( + const delivery = sendPtyDataToRenderer( + id, + makePtyDataPayload( id, - makePtyDataPayload( - id, - chunk, - pending.startSeq, - pending.containsBackgroundOutput, - pending.rawLength, - pending.transformed + chunk, + pending.startSeq, + pending.containsBackgroundOutput, + pending.rawLength, + pending.transformed + ), + pending.projectionAdmissionIds + ) + if (nextPending) { + updatePendingProjectionAdmissions( + nextPending, + propagatePendingProjectionRemainder( + nextPending, + delivery, + pendingProjectionAdmissionOptions() ) ) - ) { + } + if (!delivery.sent) { sendFailed = true break } @@ -2781,27 +2923,45 @@ export function registerPtyHandlers( return true } - function sendPtyExitToRenderer(payload: { id: string; code: number }): void { + function preparePtyExitForRenderer(payload: { id: string; code: number }): (() => void) | null { if (mainWindow.isDestroyed()) { - return + sshOutputIntake?.transferPtyProjections(payload.id, 'renderer-destroyed') + return () => {} } if (rendererExitingPtyIds.has(payload.id)) { - return + return null } rendererExitingPtyIds.add(payload.id) + let released = false + const release = (): void => { + if (released) { + return + } + released = true + rendererExitingPtyIds.delete(payload.id) + } try { - const hadReleasableRendererCredit = getRendererInFlightCharsForPty(payload.id) > 0 + if (!rendererCreditBeforeExitByPty.has(payload.id)) { + rendererCreditBeforeExitByPty.set( + payload.id, + getRendererInFlightCharsForPty(payload.id) > 0 + ) + } // Why flush before exit: the renderer tears down the terminal on pty:exit, so any batched output not yet flushed would be silently lost. const remaining = pendingData.delete(payload.id) clearFlushTimerIfIdle() if (remaining) { if (remaining.droppedOutput === true) { // Sentinel entry: only salvaged query bytes remain; keep the flag so the renderer knows the span was dropped. - sendPtyDataToRenderer(payload.id, { - id: payload.id, - data: remaining.data, - droppedOutput: true - }) + sendPtyDataToRenderer( + payload.id, + { + id: payload.id, + data: remaining.data, + droppedOutput: true + }, + remaining.projectionAdmissionIds + ) } else { sendPtyDataToRenderer( payload.id, @@ -2812,35 +2972,63 @@ export function registerPtyHandlers( remaining.containsBackgroundOutput, remaining.rawLength, remaining.transformed - ) + ), + remaining.projectionAdmissionIds ) } } - // Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id. - producerFlowControl.release(payload.id) - pendingOverflowMarkedPtys.delete(payload.id) - rendererDeliveryRestoreNeededPtys.delete(payload.id) - lastInputAtByPty.delete(payload.id) - interactiveOutputCharsByPty.delete(payload.id) - const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id) - rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit) - // Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides. - rendererDeliveryAccountingByPty.delete(payload.id) - if (hadReleasableRendererCredit) { - if (pendingDataFlushActive) { - // Why: let the open round coalesce this wake into its one post-round continuation. - const reactivatedBlocked = pendingData.reactivateBlocked() - pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked - } else { - schedulePendingDataAfterCreditReport(true) - } + return release + } catch (error) { + release() + throw error + } + } + + function finalizePtyExitForRenderer(payload: { id: string; code: number }): void { + if (mainWindow.isDestroyed()) { + rendererCreditBeforeExitByPty.delete(payload.id) + return + } + const hadReleasableRendererCredit = + rendererCreditBeforeExitByPty.get(payload.id) ?? + getRendererInFlightCharsForPty(payload.id) > 0 + rendererCreditBeforeExitByPty.delete(payload.id) + // Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id. + producerFlowControl.release(payload.id) + sourceCreditPendingPtys.delete(payload.id) + pendingOverflowMarkedPtys.delete(payload.id) + rendererDeliveryRestoreNeededPtys.delete(payload.id) + lastInputAtByPty.delete(payload.id) + interactiveOutputCharsByPty.delete(payload.id) + const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id) + rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit) + // Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides. + rendererDeliveryAccountingByPty.delete(payload.id) + if (hadReleasableRendererCredit) { + if (pendingDataFlushActive) { + // Why: let the open round coalesce this wake into its one post-round continuation. + const reactivatedBlocked = pendingData.reactivateBlocked() + pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked + } else { + schedulePendingDataAfterCreditReport(true) } - mainWindow.webContents.send('pty:exit', { - ...payload, - ...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {}) - }) + } + mainWindow.webContents.send('pty:exit', { + ...payload, + ...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {}) + }) + } + + function sendPtyExitToRenderer(payload: { id: string; code: number }): void { + const release = preparePtyExitForRenderer(payload) + if (!release) { + return + } + try { + sshOutputIntake?.transferPtyProjections(payload.id, 'legacy-pty-exit') + finalizePtyExitForRenderer(payload) } finally { - rendererExitingPtyIds.delete(payload.id) + release() } } @@ -2850,6 +3038,231 @@ export function registerPtyHandlers( } } + function acceptPtyDataForRenderer( + payload: { + id: string + data: string + sequenceChars?: number + transformed?: boolean + }, + outputSeq: number | undefined, + projection?: LegacySshProjectionSemantics + ): void { + const rawLength = payload.sequenceChars ?? payload.data.length + const preservesSeq = !payload.transformed && rawLength === payload.data.length + const startSeq = typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined + const projectionId = projection?.identity.projectionSemanticsId + if (mainWindow.isDestroyed()) { + if (projectionId) { + sshOutputIntake?.transferProjections([projectionId], 'renderer-destroyed') + } + if (flushTimer) { + clearTimeout(flushTimer) + flushTimer = null + } + producerFlowControl.releaseAll() + clearDeliveryResyncProbe() + clearPendingPtyData() + pendingOverflowMarkedPtys.clear() + rendererDeliveryAccountingByPty.clear() + rendererInFlightTotalChars = 0 + clearDispatcherReadyWatchdog() + return + } + if (rendererExitingPtyIds.has(payload.id)) { + if (projectionId) { + sshOutputIntake?.transferProjections([projectionId], 'pty-exiting') + } + return + } + if (shouldDropHiddenRendererPtyData(payload.id, getSettings?.())) { + if (projectionId) { + sshOutputIntake?.transferProjections([projectionId], 'hidden-drop') + } + const droppedChars = projection ? rawLength : payload.data.length + const drop = recordHiddenRendererPtyDataDrop(payload.id, droppedChars) + warnIfDroppingHiddenBytesForVisiblePty(payload.id, droppedChars) + if (drop.shouldEmitRestoreMarker) { + sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq) + } + return + } + if (payload.data.length === 0 && !payload.transformed) { + if (projectionId) { + sshOutputIntake?.transferProjections([projectionId], 'empty-projection') + } + return + } + const containsBackgroundOutput = + rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id) + if (containsBackgroundOutput) { + markHiddenRendererResizeOutputDelivered(payload.id) + } + const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id) + if (projection?.desktopSpan) { + sourceCreditPendingPtys.add(payload.id) + } + const pending = appendPendingPtyData( + payload.id, + pendingData.get(payload.id), + payload.data, + startSeq, + preservesSeq, + containsBackgroundOutput, + rawLength, + payload.transformed === true, + projectionId + ) + const shouldEmitPendingCapRestoreMarker = + pending.droppedOutput === true && + !overflowMarkedBeforeAppend && + pendingOverflowMarkedPtys.has(payload.id) + const nextData = pending.data + getDroppedMode2031RendererData(pending) + const isInteractiveOutput = shouldSendInteractiveOutputNow( + payload.id, + nextData, + performance.now() + ) + if (isInteractiveOutput && rendererPtyDispatcherReady) { + if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) { + setPendingPtyData(payload.id, pending) + if (shouldEmitPendingCapRestoreMarker) { + sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq) + } + updateProducerFlowControl(payload.id) + requestDeliveryResyncForGatedPty() + return + } + deletePendingPtyData(payload.id) + clearFlushTimerIfIdle() + if (shouldEmitPendingCapRestoreMarker) { + sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq) + } + pendingOverflowMarkedPtys.delete(payload.id) + try { + sendPtyDataToRenderer( + payload.id, + { + id: payload.id, + data: nextData, + ...(typeof pending.startSeq === 'number' + ? { + seq: pending.startSeq + (pending.rawLength ?? nextData.length), + rawLength: pending.rawLength ?? nextData.length + } + : {}), + ...(pending.transformed ? { transformed: true } : {}), + ...(pending.containsBackgroundOutput === true ? { background: true } : {}), + ...(pending.droppedOutput === true ? { droppedOutput: true } : {}) + }, + pending.projectionAdmissionIds + ) + } finally { + updateProducerFlowControl(payload.id) + } + return + } + setPendingPtyData(payload.id, pending) + if (shouldEmitPendingCapRestoreMarker) { + sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq) + } + updateProducerFlowControl(payload.id) + if ( + !canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) }) + ) { + requestDeliveryResyncForGatedPty() + } + if (!flushTimer) { + schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS) + } + } + + sshOutputIntakeCleanup?.() + sshOutputIntake = new SshPtyOutputIntake({ + getModelSequence: (id) => runtime?.getPtyOutputSequence(id) ?? 0, + acceptModel: (event, projection) => { + if (!runtime) { + throw new Error('SSH PTY output requires the main terminal model') + } + return runtime.acceptPtyDataBounded( + event.id, + event.data, + Date.now(), + event.rawLength, + event.transformed, + projection.desktopSpan ? [projection.desktopSpan] : undefined + ) + }, + project: (event, projection) => + acceptPtyDataForRenderer( + { + id: event.id, + data: event.data, + sequenceChars: event.rawLength, + transformed: event.transformed + }, + projection.identity.sequenceEnd, + projection + ), + prepareExit: (event) => { + const release = preparePtyExitForRenderer(event) + if (!release) { + throw new Error('pty_renderer_exit_in_progress') + } + return release + }, + finalizeExit: (event) => { + runtime?.onPtyExit(event.id, event.code, event.ptyIncarnation) + finalizePtyExitForRenderer(event) + }, + pauseProvider: (generation, id) => { + const provider = sshProvidersByGeneration.get(generation) as + | (IPtyProvider & { hasPtyDeliveryPauseAdapter?: () => boolean }) + | undefined + if (!provider?.hasPtyDeliveryPauseAdapter?.()) { + return false + } + provider.pauseProducer?.(id) + return true + }, + resumeProvider: (generation, id) => + sshProvidersByGeneration.get(generation)?.resumeProducer?.(id), + closeProvider: (generation, reason) => { + const provider = sshProvidersByGeneration.get(generation) + ;( + provider as (IPtyProvider & { closeOutputIntake?: (reason: string) => void }) | undefined + )?.closeOutputIntake?.(reason) + }, + resetModelForMigration: (_generation, id) => runtime?.resetPtyModelAfterMigrationFailure(id), + onGenerationClosed: (providerGeneration) => { + for (const id of pendingData.keys()) { + const pending = pendingData.get(id) + if ( + pending?.projectionAdmissionIds && + sshOutputIntake?.hasProjectionFromGeneration( + pending.projectionAdmissionIds, + providerGeneration + ) + ) { + pendingData.delete(id) + updateProducerFlowControl(id) + pendingOverflowMarkedPtys.delete(id) + } + } + sshProvidersByGeneration.delete(providerGeneration) + }, + publishSourceAck: publishSshPtySourceAck, + cancelSourceDelivery: cancelSshPtySourceDelivery + }) + runtime?.setRemoteTerminalSourceRangeConsumerHooks?.( + sshOutputIntake.getRemoteSourceRangeConsumerHooks() + ) + const cleanupSshOutputIntakeRegistry = installSshPtyOutputIntake(sshOutputIntake) + sshOutputIntakeCleanup = () => { + runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(null) + cleanupSshOutputIntakeRegistry() + } + async function shutdownProviderAndDetectExit( provider: IPtyProvider, id: string, @@ -2933,120 +3346,7 @@ export function registerPtyHandlers( const outputSeq = isLocalProvider ? runtime?.getPtyOutputSequence(payload.id) : runtime?.onPtyData(payload.id, payload.data, Date.now(), rawLength, payload.transformed) - const rendererData = payload.data - const preservesSeq = !payload.transformed && rawLength === payload.data.length - const startSeq = - typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined - if (mainWindow.isDestroyed()) { - // Why clear the flush timer: macOS app re-activation otherwise leaks orphaned timers from the previous window's registration. - if (flushTimer) { - clearTimeout(flushTimer) - flushTimer = null - } - producerFlowControl.releaseAll() - clearDeliveryResyncProbe() - clearPendingPtyData() - pendingOverflowMarkedPtys.clear() - rendererDeliveryAccountingByPty.clear() - rendererInFlightTotalChars = 0 - clearDispatcherReadyWatchdog() - return - } - if (rendererExitingPtyIds.size > 0 && rendererExitingPtyIds.has(payload.id)) { - return - } - const settings = getSettings?.() - // Why drop before the interactive bypass: runtime already ingested the chunk, so gated PTYs skip both renderer paths and reveal restores from the snapshot. - if (shouldDropHiddenRendererPtyData(payload.id, settings)) { - const drop = recordHiddenRendererPtyDataDrop(payload.id, payload.data.length) - warnIfDroppingHiddenBytesForVisiblePty(payload.id, payload.data.length) - if (drop.shouldEmitRestoreMarker) { - sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq) - } - return - } - if (rendererData.length === 0 && !payload.transformed) { - return - } - const containsBackgroundOutput = - rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id) - if (containsBackgroundOutput) { - markHiddenRendererResizeOutputDelivered(payload.id) - } - const existing = pendingData.get(payload.id) - const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id) - const pending = appendPendingPtyData( - payload.id, - existing, - rendererData, - startSeq, - preservesSeq, - containsBackgroundOutput, - rawLength, - payload.transformed === true - ) - const shouldEmitPendingCapRestoreMarker = - pending.droppedOutput === true && - !overflowMarkedBeforeAppend && - pendingOverflowMarkedPtys.has(payload.id) - const nextData = pending.data + getDroppedMode2031RendererData(pending) - const isInteractiveOutput = shouldSendInteractiveOutputNow( - payload.id, - nextData, - performance.now() - ) - // Why gate the fast path on the handshake too: else boot-window keystroke echo is sent into a listener-less page and pins the gate. - if (isInteractiveOutput && rendererPtyDispatcherReady) { - // Why the reserve: keep input echo from being pinned behind unrelated bulk output; it's bounded and the per-PTY cap still prevents an active TUI runaway. - if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) { - setPendingPtyData(payload.id, pending) - if (shouldEmitPendingCapRestoreMarker) { - sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq) - } - updateProducerFlowControl(payload.id) - requestDeliveryResyncForGatedPty() - return - } - deletePendingPtyData(payload.id) - clearFlushTimerIfIdle() - if (shouldEmitPendingCapRestoreMarker) { - sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq) - } - pendingOverflowMarkedPtys.delete(payload.id) - // Why immediate: agent TUIs redraw small prompt regions per keystroke; the throughput batch timer would add visible input latency. - try { - sendPtyDataToRenderer(payload.id, { - id: payload.id, - data: nextData, - ...(typeof pending.startSeq === 'number' - ? { - seq: pending.startSeq + (pending.rawLength ?? nextData.length), - rawLength: pending.rawLength ?? nextData.length - } - : {}), - ...(pending.transformed ? { transformed: true } : {}), - ...(pending.containsBackgroundOutput === true ? { background: true } : {}), - ...(pending.droppedOutput === true ? { droppedOutput: true } : {}) - }) - } finally { - updateProducerFlowControl(payload.id) - } - return - } - setPendingPtyData(payload.id, pending) - if (shouldEmitPendingCapRestoreMarker) { - sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq) - } - updateProducerFlowControl(payload.id) - // Why probe on data arrival (not flush skips): new output for a fully gated PTY is the moment stuck delivery becomes observable. - if ( - !canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) }) - ) { - requestDeliveryResyncForGatedPty() - } - if (!flushTimer) { - schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS) - } + acceptPtyDataForRenderer(payload, outputSeq) }) localExitUnsub = localProvider.onExit((payload) => { if (!isCurrentPtyExit(payload)) { @@ -5646,6 +5946,9 @@ export function registerPtyHandlers( const pending = pendingData.get(args.id) if (pending && transition.droppable) { pendingData.delete(args.id) + if (pending.projectionAdmissionIds) { + sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop') + } updateProducerFlowControl(args.id) pendingOverflowMarkedPtys.delete(args.id) const drop = recordHiddenRendererPtyDataDrop(args.id, pending.data.length) diff --git a/src/main/ipc/ssh-pty-closed-generation-ranges.ts b/src/main/ipc/ssh-pty-closed-generation-ranges.ts new file mode 100644 index 00000000000..454ca5b2815 --- /dev/null +++ b/src/main/ipc/ssh-pty-closed-generation-ranges.ts @@ -0,0 +1,53 @@ +type ClosedGenerationRange = { + start: number + end: number +} + +export class SshPtyClosedGenerationRanges { + private readonly ranges: ClosedGenerationRange[] = [] + + add(generation: number): void { + let index = 0 + while (index < this.ranges.length && this.ranges[index]!.end + 1 < generation) { + index++ + } + const current = this.ranges[index] + if (!current || generation + 1 < current.start) { + this.ranges.splice(index, 0, { start: generation, end: generation }) + return + } + current.start = Math.min(current.start, generation) + current.end = Math.max(current.end, generation) + const next = this.ranges[index + 1] + if (next && current.end + 1 >= next.start) { + current.end = Math.max(current.end, next.end) + this.ranges.splice(index + 1, 1) + } + } + + has(generation: number): boolean { + for (const range of this.ranges) { + if (generation < range.start) { + return false + } + if (generation <= range.end) { + return true + } + } + return false + } + + get size(): number { + return this.ranges.length + } + + get activeGaps(): number { + const highWater = this.ranges.at(-1)?.end ?? 0 + let closedGenerations = 0 + for (const range of this.ranges) { + closedGenerations += range.end - range.start + 1 + } + // Why: provider generations allocate from 1, so unclosed IDs below high-water remain active. + return highWater - closedGenerations + } +} diff --git a/src/main/ipc/ssh-pty-legacy-projection-publication.ts b/src/main/ipc/ssh-pty-legacy-projection-publication.ts new file mode 100644 index 00000000000..395b9233d60 --- /dev/null +++ b/src/main/ipc/ssh-pty-legacy-projection-publication.ts @@ -0,0 +1,115 @@ +import { + projectionError, + reclaimProjectionRecord, + type DesktopProjectionSpan, + type ProjectionRecord +} from './ssh-pty-legacy-projection-record' + +export function publishLegacyProjectionPrefix( + records: Map, + ids: readonly string[], + displayChars: number, + accountingChars: number +): void { + let displayRemaining = Math.max(0, displayChars) + let accountingRemaining = Math.max(0, accountingChars) + for (const id of ids) { + const record = records.get(id) + if (!record) { + continue + } + const displayLength = + record.semantics.identity.displayEnd - record.semantics.identity.displayStart + const unpublishedDisplay = displayLength - record.publishedDisplay + const unpublishedAccounting = record.semantics.identity.rawLength - record.publishedAccounting + if (displayLength === 0 && unpublishedAccounting > 0) { + const publishAccounting = Math.min(accountingRemaining, unpublishedAccounting) + if (publishAccounting !== unpublishedAccounting) { + throw projectionError('ssh_projection_indivisible_split') + } + record.publishedAccounting += publishAccounting + record.state = 'published' + accountingRemaining -= publishAccounting + continue + } + if (unpublishedDisplay <= 0) { + continue + } + const publishDisplay = Math.min(displayRemaining, unpublishedDisplay) + if (publishDisplay <= 0) { + break + } + if (record.semantics.identity.transformed && publishDisplay !== unpublishedDisplay) { + throw projectionError('ssh_projection_indivisible_split') + } + const publishAccounting = + publishDisplay === unpublishedDisplay + ? Math.min(accountingRemaining, unpublishedAccounting) + : publishDisplay + record.publishedDisplay += publishDisplay + record.publishedAccounting += publishAccounting + record.state = 'published' + displayRemaining -= publishDisplay + accountingRemaining -= publishAccounting + } + if (displayRemaining !== 0 || accountingRemaining !== 0) { + throw projectionError('ssh_projection_publish_range_mismatch') + } +} + +export function hasUnpublishedLegacyProjection( + records: ReadonlyMap, + id: string +): boolean { + const record = records.get(id) + if (!record || record.state === 'reserved') { + return false + } + const displayLength = + record.semantics.identity.displayEnd - record.semantics.identity.displayStart + return ( + record.publishedDisplay < displayLength || + record.publishedAccounting < record.semantics.identity.rawLength + ) +} + +export function settlePublishedLegacyProjectionPrefix( + records: Map, + idsByPty: Map, + ptyId: string, + accountingChars: number, + onSettled: ((span: DesktopProjectionSpan, reason: string) => void) | undefined +): { settled: number; completed: number } { + let remaining = Math.max(0, accountingChars) + let settled = 0 + let completed = 0 + for (const id of idsByPty.get(ptyId)?.slice() ?? []) { + const record = records.get(id) + if (!record) { + continue + } + const available = record.publishedAccounting - record.settledAccounting + if (available <= 0) { + continue + } + const take = Math.min(remaining, available) + const finishes = + record.settledAccounting + take === record.semantics.identity.rawLength && + record.publishedDisplay === + record.semantics.identity.displayEnd - record.semantics.identity.displayStart + if (finishes && record.semantics.desktopSpan) { + onSettled?.(record.semantics.desktopSpan, 'renderer-parse') + } + record.settledAccounting += take + settled += take + remaining -= take + if (finishes) { + completed++ + reclaimProjectionRecord(records, idsByPty, id, ptyId) + } + if (remaining === 0) { + break + } + } + return { settled, completed } +} diff --git a/src/main/ipc/ssh-pty-legacy-projection-record.ts b/src/main/ipc/ssh-pty-legacy-projection-record.ts new file mode 100644 index 00000000000..45c666dc09a --- /dev/null +++ b/src/main/ipc/ssh-pty-legacy-projection-record.ts @@ -0,0 +1,218 @@ +import type { + Mode2031ReplyDecision, + Mode2031ReplyScanState +} from '../../shared/terminal-color-scheme-protocol' +import { INITIAL_MODE_2031_REPLY_SCAN_STATE } from '../../shared/terminal-color-scheme-protocol' +import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range' + +export type LegacySshProjectionIdentity = Readonly<{ + projectionSemanticsId: string + ptyId: string + providerGeneration: number + ptyIncarnation: string + displayStart: number + displayEnd: number + sequenceEnd: number + rawLength: number + transformed: boolean +}> + +export type LegacySshProjectionSemantics = Readonly<{ + identity: LegacySshProjectionIdentity + desktopSpan?: DesktopProjectionSpan + beforeScanner: Readonly + afterScanner: Readonly + decision: Mode2031ReplyDecision +}> + +export type DesktopProjectionSpan = Readonly< + TerminalOutputSourceRange & { + projectionSemanticsId: string + transform: Readonly<{ + transformed: boolean + rawLengthSu: number + scalarSafe: boolean + }> + } +> + +export type ProjectionState = 'reserved' | 'committed' | 'published' + +export type ProjectionRecord = { + semantics: LegacySshProjectionSemantics + state: ProjectionState + publishedDisplay: number + publishedAccounting: number + settledAccounting: number +} + +export type PtyProjectionCursor = { + providerGeneration: number + ptyIncarnation: string + displayEnd: number + scanner: Mode2031ReplyScanState +} + +export type LegacySshProjectionReservation = Readonly<{ + semantics: LegacySshProjectionSemantics +}> + +export type LegacySshProjectionDebugSnapshot = { + reserved: number + committed: number + published: number + settled: number + transferred: number + rolledBack: number + records: number + cursors: number +} + +export function scannerSnapshot(state: Mode2031ReplyScanState): Readonly { + return Object.freeze({ tail: state.tail, pendingSubscribe: state.pendingSubscribe }) +} + +export function resetProjectionCursorForGap( + cursors: ReadonlyMap, + ptyId: string +): void { + const cursor = cursors.get(ptyId) + if (cursor) { + cursor.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE } + } +} + +export function projectionError(code: string): Error { + return Object.assign(new Error(code), { code }) +} + +export function projectionDebugSnapshot( + records: ReadonlyMap, + cursors: ReadonlyMap, + terminalCounts: { settled: number; transferred: number; rolledBack: number } +): LegacySshProjectionDebugSnapshot { + const result = { + reserved: 0, + committed: 0, + published: 0, + ...terminalCounts, + records: records.size, + cursors: cursors.size + } + for (const record of records.values()) { + result[record.state]++ + } + return result +} + +export function reclaimProjectionRecord( + records: Map, + idsByPty: Map, + id: string, + ptyId: string +): void { + records.delete(id) + const ids = idsByPty.get(ptyId) + if (!ids) { + return + } + const index = ids.indexOf(id) + if (index >= 0) { + ids.splice(index, 1) + } + if (ids.length === 0) { + idsByPty.delete(ptyId) + } +} + +export function requireProjectionRecord( + records: ReadonlyMap, + id: string +): ProjectionRecord { + const record = records.get(id) + if (!record) { + throw projectionError('ssh_projection_reservation_missing') + } + return record +} + +export function rollbackCommittedProjectionRecord( + records: Map, + idsByPty: Map, + cursors: Map, + reservation: LegacySshProjectionReservation +): string | null { + const id = reservation.semantics.identity.projectionSemanticsId + const record = records.get(id) + if (!record || record.state !== 'committed') { + return null + } + const { identity, beforeScanner } = record.semantics + const cursor = cursors.get(identity.ptyId) + if ( + idsByPty.get(identity.ptyId)?.at(-1) !== id || + !cursor || + cursor.providerGeneration !== identity.providerGeneration || + cursor.ptyIncarnation !== identity.ptyIncarnation || + cursor.displayEnd !== identity.displayEnd + ) { + return null + } + cursor.displayEnd = identity.displayStart + cursor.scanner = { ...beforeScanner } + reclaimProjectionRecord(records, idsByPty, id, identity.ptyId) + return identity.ptyId +} + +export function closeProjectionPty( + cursors: Map, + ptyId: string, + providerGeneration: number, + ptyIncarnation: string, + beforeDelete: () => void +): void { + const cursor = cursors.get(ptyId) + if ( + !cursor || + cursor.providerGeneration !== providerGeneration || + cursor.ptyIncarnation !== ptyIncarnation + ) { + return + } + beforeDelete() + cursors.delete(ptyId) +} + +export function getOrCreateProjectionCursor( + cursors: Map, + args: { ptyId: string; providerGeneration: number; ptyIncarnation: string }, + replaceGeneration: (providerGeneration: number) => void +): PtyProjectionCursor { + const existing = cursors.get(args.ptyId) + if (!existing) { + const cursor = { + providerGeneration: args.providerGeneration, + ptyIncarnation: args.ptyIncarnation, + displayEnd: 0, + scanner: { ...INITIAL_MODE_2031_REPLY_SCAN_STATE } + } + cursors.set(args.ptyId, cursor) + return cursor + } + if (args.providerGeneration < existing.providerGeneration) { + throw projectionError('ssh_projection_stale_generation') + } + if ( + args.providerGeneration === existing.providerGeneration && + args.ptyIncarnation !== existing.ptyIncarnation + ) { + throw projectionError('ssh_projection_stale_incarnation') + } + if (args.providerGeneration > existing.providerGeneration) { + replaceGeneration(existing.providerGeneration) + existing.providerGeneration = args.providerGeneration + existing.ptyIncarnation = args.ptyIncarnation + existing.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE } + } + return existing +} diff --git a/src/main/ipc/ssh-pty-legacy-projection.test.ts b/src/main/ipc/ssh-pty-legacy-projection.test.ts new file mode 100644 index 00000000000..e926456619a --- /dev/null +++ b/src/main/ipc/ssh-pty-legacy-projection.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it } from 'vitest' +import { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection' +import { SshPtyProjectionTerminality } from './ssh-pty-projection-terminality' + +function reserve( + ledger: SshPtyLegacyProjectionLedger, + overrides: Partial[0]> = {} +) { + return ledger.reserve({ + ptyId: 'pty-1', + providerGeneration: 3, + ptyIncarnation: 'incarnation-1', + data: 'abc', + sequenceEnd: 3, + rawLength: 3, + transformed: false, + ...overrides + }) +} + +function source(sourceStartSu: number, sourceEndSu: number, deliveryToken = 'token-1') { + return { + spanId: `span-${sourceStartSu}`, + clientGeneration: 2, + ownerGeneration: 4, + deliveryToken, + sourceStartSu, + sourceEndSu + } +} + +describe('SshPtyLegacyProjectionLedger', () => { + it('drains exact terminality waiters when their provider generation closes', async () => { + const terminality = new SshPtyProjectionTerminality() + const terminal = terminality.whenTerminal('pty-1', 3, 'incarnation-1', () => true) + let nextResolved = false + const next = terminality + .whenTerminal('pty-1', 4, 'incarnation-2', () => true) + .then(() => { + nextResolved = true + }) + + terminality.closeGeneration(3) + + await expect(terminal).resolves.toBeUndefined() + expect(nextResolved).toBe(false) + terminality.closeGeneration(4) + await next + }) + + it('rolls back scanner and display reservations before commit', () => { + const ledger = new SshPtyLegacyProjectionLedger() + const partial = reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 }) + expect(ledger.rollback(partial)).toBe(true) + + const next = reserve(ledger, { data: '31h', sequenceEnd: 3 }) + expect(next.semantics.identity.displayStart).toBe(0) + expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false }) + expect(next.semantics.decision).toBeNull() + }) + + it('keeps immutable generation, incarnation, display, sequence, raw length, and scanner facts', () => { + const ledger = new SshPtyLegacyProjectionLedger() + const first = reserve(ledger, { + data: '\x1b[?2031h', + rawLength: 11, + sequenceEnd: 11 + }) + const semantics = ledger.commit(first) + + expect(semantics.identity).toMatchObject({ + providerGeneration: 3, + ptyIncarnation: 'incarnation-1', + displayStart: 0, + displayEnd: 8, + sequenceEnd: 11, + rawLength: 11 + }) + expect(semantics.decision).toBe('subscribed') + expect(Object.isFrozen(semantics)).toBe(true) + expect(Object.isFrozen(semantics.identity)).toBe(true) + }) + + it('rejects stale generations and resets cross-chunk scanner state on gaps', () => { + const ledger = new SshPtyLegacyProjectionLedger() + ledger.commit(reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 })) + ledger.resetForGap('pty-1') + const next = reserve(ledger, { data: '31h', sequenceEnd: 8 }) + expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false }) + + expect(() => reserve(ledger, { providerGeneration: 2 })).toThrow( + 'ssh_projection_stale_generation' + ) + }) + + it('publishes, settles, and transfers explicit ranges', () => { + const ledger = new SshPtyLegacyProjectionLedger() + const first = ledger.commit(reserve(ledger)) + ledger.publishPrefix([first.identity.projectionSemanticsId], 3, 3) + expect(ledger.settlePublishedPrefix('pty-1', 2)).toBe(2) + expect(ledger.transfer([first.identity.projectionSemanticsId], 'renderer-reload')).toBe(1) + expect(ledger.getDebugSnapshot()).toMatchObject({ transferred: 1, records: 0 }) + }) + + it('resolves exact PTY terminal waiters only after settlement or transfer', async () => { + const ledger = new SshPtyLegacyProjectionLedger() + const projection = ledger.commit(reserve(ledger)) + ledger.publishPrefix([projection.identity.projectionSemanticsId], 3, 3) + + let settled = false + const terminal = ledger.whenPtyTerminal('pty-1', 3, 'incarnation-1').then(() => { + settled = true + }) + await Promise.resolve() + expect(settled).toBe(false) + + ledger.settlePublishedPrefix('pty-1', 2) + await Promise.resolve() + expect(settled).toBe(false) + ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload') + await terminal + expect(settled).toBe(true) + }) + + it('publishes and settles transformed source accounting with no display text', () => { + const ledger = new SshPtyLegacyProjectionLedger() + const projection = ledger.commit( + reserve(ledger, { + data: '', + sequenceEnd: 9, + rawLength: 9, + transformed: true + }) + ) + + ledger.publishPrefix([projection.identity.projectionSemanticsId], 0, 9) + expect(ledger.settlePublishedPrefix('pty-1', 9)).toBe(9) + expect(ledger.getDebugSnapshot()).toMatchObject({ settled: 1, records: 0 }) + }) + + it('reclaims a closed PTY cursor so its id can be reused by a new incarnation', () => { + const ledger = new SshPtyLegacyProjectionLedger() + ledger.commit(reserve(ledger)) + ledger.closePty('pty-1', 3, 'incarnation-1', 'pty-exit') + + const next = reserve(ledger, { + ptyIncarnation: 'incarnation-2', + data: 'next', + sequenceEnd: 4, + rawLength: 4 + }) + expect(next.semantics.identity).toMatchObject({ + ptyIncarnation: 'incarnation-2', + displayStart: 0 + }) + expect(ledger.getDebugSnapshot()).toMatchObject({ records: 1, cursors: 1 }) + }) + + it('keeps split publication attached to one immutable desktop span', () => { + const settled: unknown[] = [] + const ledger = new SshPtyLegacyProjectionLedger({ + onSettled: (span) => settled.push(span) + }) + const projection = ledger.commit( + reserve(ledger, { + data: 'abcd', + rawLength: 4, + sequenceEnd: 4, + source: source(0, 4) + }) + ) + const id = projection.identity.projectionSemanticsId + expect(ledger.hasUnpublished(id)).toBe(true) + + ledger.publishPrefix([id], 2, 2) + expect(ledger.hasUnpublished(id)).toBe(true) + ledger.settlePublishedPrefix('pty-1', 2) + expect(settled).toEqual([]) + ledger.publishPrefix([id], 2, 2) + expect(ledger.hasUnpublished(id)).toBe(false) + ledger.settlePublishedPrefix('pty-1', 2) + + expect(settled).toEqual([ + expect.objectContaining({ + spanId: 'span-0', + projectionSemanticsId: id, + sourceStartSu: 0, + sourceEndSu: 4, + displayStart: 0, + displayEnd: 4 + }) + ]) + }) + + it('preserves scanner and display facts after transfer and delivery-token replacement', () => { + const transferred: unknown[] = [] + const ledger = new SshPtyLegacyProjectionLedger({ + onTransferred: (span) => transferred.push(span) + }) + const partial = ledger.commit( + reserve(ledger, { + data: '\x1b[?20', + rawLength: 5, + sequenceEnd: 5, + source: source(0, 5) + }) + ) + ledger.transfer([partial.identity.projectionSemanticsId], 'renderer-reload') + const continuation = reserve(ledger, { + data: '31h', + rawLength: 3, + sequenceEnd: 8, + source: source(5, 8, 'token-2') + }) + + expect(continuation.semantics.identity.displayStart).toBe(5) + expect(continuation.semantics.beforeScanner.tail).toBe('\x1b[?20') + expect(continuation.semantics.decision).toBe('subscribed') + expect(transferred).toEqual([ + expect.objectContaining({ + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 5 + }) + ]) + }) + + it('leaves a projection intact when its terminal transition cannot commit', () => { + const ledger = new SshPtyLegacyProjectionLedger({ + onTransferred: () => { + throw new Error('replacement unavailable') + } + }) + const projection = ledger.commit(reserve(ledger, { source: source(0, 3) })) + + expect(() => + ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload') + ).toThrow('replacement unavailable') + expect(ledger.getDebugSnapshot()).toMatchObject({ + transferred: 0, + records: 1 + }) + }) +}) diff --git a/src/main/ipc/ssh-pty-legacy-projection.ts b/src/main/ipc/ssh-pty-legacy-projection.ts new file mode 100644 index 00000000000..0e779a82b4f --- /dev/null +++ b/src/main/ipc/ssh-pty-legacy-projection.ts @@ -0,0 +1,318 @@ +import { scanMode2031ReplyDecision } from '../../shared/terminal-color-scheme-protocol' +import { + closeProjectionPty, + getOrCreateProjectionCursor, + projectionDebugSnapshot, + projectionError, + reclaimProjectionRecord, + resetProjectionCursorForGap, + requireProjectionRecord, + rollbackCommittedProjectionRecord, + scannerSnapshot, + type DesktopProjectionSpan, + type LegacySshProjectionDebugSnapshot, + type LegacySshProjectionReservation, + type LegacySshProjectionSemantics, + type ProjectionRecord, + type PtyProjectionCursor +} from './ssh-pty-legacy-projection-record' +import { + projectionHasOpen, + resolveProjectionTerminality, + SshPtyProjectionTerminality, + unpublishedProjectionIds +} from './ssh-pty-projection-terminality' +import { + hasUnpublishedLegacyProjection, + publishLegacyProjectionPrefix, + settlePublishedLegacyProjectionPrefix +} from './ssh-pty-legacy-projection-publication' + +export type { + DesktopProjectionSpan, + LegacySshProjectionIdentity, + LegacySshProjectionReservation, + LegacySshProjectionSemantics +} from './ssh-pty-legacy-projection-record' + +export type SshPtyLegacyProjectionLedgerOptions = { + onSettled?: (span: DesktopProjectionSpan, reason: string) => void + onTransferred?: (span: DesktopProjectionSpan, reason: string) => void +} + +export class SshPtyLegacyProjectionLedger { + private nextId = 1 + private readonly records = new Map() + private readonly cursorByPty = new Map() + private readonly idsByPty = new Map() + private readonly terminality = new SshPtyProjectionTerminality() + private settledCount = 0 + private transferredCount = 0 + private rolledBackCount = 0 + + constructor(private readonly options: SshPtyLegacyProjectionLedgerOptions = {}) {} + + reserve(args: { + ptyId: string + providerGeneration: number + ptyIncarnation: string + data: string + sequenceEnd: number + rawLength: number + transformed: boolean + source?: Readonly<{ + relayPtyId?: string + spanId: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string + sourceStartSu: number + sourceEndSu: number + }> + }): LegacySshProjectionReservation { + const cursor = getOrCreateProjectionCursor(this.cursorByPty, args, (generation) => + this.transferGeneration(generation, 'provider-generation-replaced') + ) + if ( + cursor.providerGeneration !== args.providerGeneration || + cursor.ptyIncarnation !== args.ptyIncarnation + ) { + throw projectionError('ssh_projection_stale_generation') + } + const scan = scanMode2031ReplyDecision(cursor.scanner, args.data) + const projectionSemanticsId = `ssh-projection:${args.providerGeneration}:${this.nextId++}` + const identity = Object.freeze({ + projectionSemanticsId, + ptyId: args.ptyId, + providerGeneration: args.providerGeneration, + ptyIncarnation: args.ptyIncarnation, + displayStart: cursor.displayEnd, + displayEnd: cursor.displayEnd + args.data.length, + sequenceEnd: args.sequenceEnd, + rawLength: args.rawLength, + transformed: args.transformed + }) + const semantics = Object.freeze({ + identity, + ...(args.source + ? { + desktopSpan: Object.freeze({ + ...args.source, + id: args.source.relayPtyId ?? args.ptyId, + projectionSemanticsId, + providerGeneration: args.providerGeneration, + ptyIncarnation: args.ptyIncarnation, + displayStart: identity.displayStart, + displayEnd: identity.displayEnd, + splittable: !args.transformed, + transform: Object.freeze({ + transformed: args.transformed, + rawLengthSu: args.rawLength, + scalarSafe: !args.transformed + }) + }) + } + : {}), + beforeScanner: scannerSnapshot(cursor.scanner), + afterScanner: scannerSnapshot(scan.state), + decision: scan.decision + }) + this.records.set(projectionSemanticsId, { + semantics, + state: 'reserved', + publishedDisplay: 0, + publishedAccounting: 0, + settledAccounting: 0 + }) + return Object.freeze({ semantics }) + } + + commit(reservation: LegacySshProjectionReservation): LegacySshProjectionSemantics { + const record = requireProjectionRecord( + this.records, + reservation.semantics.identity.projectionSemanticsId + ) + if (record.state !== 'reserved') { + throw projectionError('ssh_projection_commit_invalid') + } + const { identity, afterScanner } = record.semantics + const cursor = this.cursorByPty.get(identity.ptyId) + if ( + !cursor || + cursor.providerGeneration !== identity.providerGeneration || + cursor.ptyIncarnation !== identity.ptyIncarnation || + cursor.displayEnd !== identity.displayStart + ) { + this.records.delete(identity.projectionSemanticsId) + throw projectionError('ssh_projection_commit_stale') + } + cursor.displayEnd = identity.displayEnd + cursor.scanner = { ...afterScanner } + record.state = 'committed' + const ids = this.idsByPty.get(identity.ptyId) ?? [] + ids.push(identity.projectionSemanticsId) + this.idsByPty.set(identity.ptyId, ids) + return record.semantics + } + + rollback(reservation: LegacySshProjectionReservation): boolean { + const id = reservation.semantics.identity.projectionSemanticsId + const record = this.records.get(id) + if (!record || record.state !== 'reserved') { + return false + } + this.records.delete(id) + this.rolledBackCount++ + return true + } + + rollbackCommitted(reservation: LegacySshProjectionReservation): boolean { + const ptyId = rollbackCommittedProjectionRecord( + this.records, + this.idsByPty, + this.cursorByPty, + reservation + ) + if (!ptyId) { + return false + } + this.rolledBackCount++ + resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId) + return true + } + + publishPrefix(ids: readonly string[], displayChars: number, accountingChars: number): void { + publishLegacyProjectionPrefix(this.records, ids, displayChars, accountingChars) + } + + settlePublishedPrefix(ptyId: string, accountingChars: number): number { + const result = settlePublishedLegacyProjectionPrefix( + this.records, + this.idsByPty, + ptyId, + accountingChars, + this.options.onSettled + ) + this.settledCount += result.completed + resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId) + return result.settled + } + + transfer(ids: readonly string[], reason: string): number { + let transferred = 0 + const touchedPtys = new Set() + for (const id of ids.slice()) { + const record = this.records.get(id) + if (!record || record.state === 'reserved') { + continue + } + const ptyId = record.semantics.identity.ptyId + if (record.semantics.desktopSpan) { + this.options.onTransferred?.(record.semantics.desktopSpan, reason) + } + this.transferredCount++ + reclaimProjectionRecord(this.records, this.idsByPty, id, ptyId) + touchedPtys.add(ptyId) + transferred++ + } + for (const ptyId of touchedPtys) { + resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId) + } + return transferred + } + + whenPtyTerminal( + ptyId: string, + providerGeneration: number, + ptyIncarnation: string + ): Promise { + return this.terminality.whenTerminal( + ptyId, + providerGeneration, + ptyIncarnation, + projectionHasOpen(this.records, this.idsByPty, ptyId) + ) + } + + transferGeneration(providerGeneration: number, reason: string): number { + const ids: string[] = [] + for (const [id, record] of this.records) { + if (record.semantics.identity.providerGeneration === providerGeneration) { + ids.push(id) + } + } + return this.transfer(ids, reason) + } + + transferPty(ptyId: string, reason: string): number { + return this.transfer(this.idsByPty.get(ptyId) ?? [], reason) + } + + transferUnpublishedPty( + ptyId: string, + providerGeneration: number, + ptyIncarnation: string, + reason: string + ): number { + const ids = unpublishedProjectionIds( + this.records, + this.idsByPty.get(ptyId) ?? [], + providerGeneration, + ptyIncarnation + ) + return this.transfer(ids, reason) + } + + closePty( + ptyId: string, + providerGeneration: number, + ptyIncarnation: string, + reason: string + ): void { + closeProjectionPty(this.cursorByPty, ptyId, providerGeneration, ptyIncarnation, () => { + this.transferPty(ptyId, reason) + this.idsByPty.delete(ptyId) + }) + } + + closeGeneration(providerGeneration: number, reason: string): void { + for (const [id, record] of this.records) { + if (record.semantics.identity.providerGeneration !== providerGeneration) { + continue + } + if (record.state === 'reserved') { + this.records.delete(id) + this.rolledBackCount++ + } else { + this.transfer([id], reason) + } + } + for (const [ptyId, cursor] of this.cursorByPty) { + if (cursor.providerGeneration === providerGeneration) { + this.cursorByPty.delete(ptyId) + this.idsByPty.delete(ptyId) + } + } + this.terminality.closeGeneration(providerGeneration) + } + + resetForGap(ptyId: string): void { + resetProjectionCursorForGap(this.cursorByPty, ptyId) + } + + get(id: string): LegacySshProjectionSemantics | undefined { + return this.records.get(id)?.semantics + } + + hasUnpublished(id: string): boolean { + return hasUnpublishedLegacyProjection(this.records, id) + } + + getDebugSnapshot(): LegacySshProjectionDebugSnapshot { + return projectionDebugSnapshot(this.records, this.cursorByPty, { + settled: this.settledCount, + transferred: this.transferredCount, + rolledBack: this.rolledBackCount + }) + } +} diff --git a/src/main/ipc/ssh-pty-model-admission-contract.ts b/src/main/ipc/ssh-pty-model-admission-contract.ts new file mode 100644 index 00000000000..7d3347ec4a1 --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-contract.ts @@ -0,0 +1,35 @@ +export type SshPtyModelAdmissionKey = Readonly<{ + ptyId: string + providerGeneration: number +}> + +export type SshPtyModelAdmissionReceipt = Readonly<{ + ptyId: string + providerGeneration: number + sequence: number +}> + +export type SshPtyModelAdmissionDebugSnapshot = Readonly<{ + sourceUnits: number + bytes: number + pressureFrames: number + pressureBytes: number + pausedPtys: number + migratingPtys: number +}> + +export type SshPtyModelAdmissionOptions = { + perPtyHighSourceUnits?: number + perPtyHighBytes?: number + perPtyLowSourceUnits?: number + perPtyLowBytes?: number + globalHighSourceUnits?: number + globalHighBytes?: number + globalLowSourceUnits?: number + globalLowBytes?: number + pressureMaxFrames?: number + pressureMaxBytes?: number + pauseProvider?: (key: SshPtyModelAdmissionKey) => boolean + resumeProvider?: (key: SshPtyModelAdmissionKey) => void + closeProvider?: (providerGeneration: number, reason: string) => void +} diff --git a/src/main/ipc/ssh-pty-model-admission-entry.ts b/src/main/ipc/ssh-pty-model-admission-entry.ts new file mode 100644 index 00000000000..a9894181e71 --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-entry.ts @@ -0,0 +1,151 @@ +import type { + SshPtyModelAdmissionKey, + SshPtyModelAdmissionReceipt +} from './ssh-pty-model-admission-contract' +import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits' + +export type AdmissionCharge = { sourceUnits: number; bytes: number } + +export type AdmissionEntry = { + key: SshPtyModelAdmissionKey + charge: AdmissionCharge + run: () => { sequence: number; completion: Promise } + resolve: (receipt: SshPtyModelAdmissionReceipt) => void + reject: (error: Error) => void + state: 'queued' | 'running' | 'pressure' | 'settled' +} + +export type PtyUsage = AdmissionCharge & { + queued: AdmissionEntry[] + running: AdmissionEntry | null +} + +export function admissionError(code: string): Error { + return Object.assign(new Error(code), { code }) +} + +export function retainedBytes(data: string): number { + return Math.max(Buffer.byteLength(data, 'utf8'), 2 * data.length) + 128 +} + +export function admissionKeyId(key: SshPtyModelAdmissionKey): string { + return `${key.providerGeneration}\0${key.ptyId}` +} + +export function canReserveAdmission(args: { + key: SshPtyModelAdmissionKey + charge: AdmissionCharge + limits: SshPtyModelAdmissionLimits + usageByPty: ReadonlyMap + closingGenerations: ReadonlySet + globalSourceUnits: number + globalBytes: number +}): boolean { + if (args.closingGenerations.has(args.key.providerGeneration)) { + return false + } + const usage = args.usageByPty.get(admissionKeyId(args.key)) + return ( + (usage?.sourceUnits ?? 0) + args.charge.sourceUnits <= args.limits.perPtyHighSourceUnits && + (usage?.bytes ?? 0) + args.charge.bytes <= args.limits.perPtyHighBytes && + args.globalSourceUnits + args.charge.sourceUnits <= args.limits.globalHighSourceUnits && + args.globalBytes + args.charge.bytes <= args.limits.globalHighBytes + ) +} + +export function pressureHasAdmissionKey( + entries: readonly AdmissionEntry[], + key: SshPtyModelAdmissionKey +): boolean { + const id = admissionKeyId(key) + return entries.some((entry) => admissionKeyId(entry.key) === id) +} + +export function takePressureEntriesForGeneration( + entries: AdmissionEntry[], + providerGeneration: number +): AdmissionEntry[] { + const removed: AdmissionEntry[] = [] + for (let index = entries.length - 1; index >= 0; index--) { + if (entries[index]!.key.providerGeneration === providerGeneration) { + removed.unshift(entries.splice(index, 1)[0]!) + } + } + return removed +} + +export function cancelAdmissionGeneration(args: { + pressure: AdmissionEntry[] + usageByPty: Map + idleWaiters: Map void>> + providerGeneration: number + error: Error + release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void +}): number { + let pressureBytes = 0 + for (const entry of takePressureEntriesForGeneration(args.pressure, args.providerGeneration)) { + pressureBytes += entry.charge.bytes + entry.state = 'settled' + entry.reject(args.error) + } + for (const [id, usage] of args.usageByPty) { + const canceled = usage.queued.filter( + (entry) => entry.key.providerGeneration === args.providerGeneration + ) + usage.queued = usage.queued.filter( + (entry) => entry.key.providerGeneration !== args.providerGeneration + ) + if (usage.running?.key.providerGeneration === args.providerGeneration) { + canceled.push(usage.running) + usage.running = null + } + for (const entry of canceled) { + if (entry.state === 'settled') { + continue + } + entry.state = 'settled' + args.release(entry.key, entry.charge) + entry.reject(args.error) + } + if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) { + args.usageByPty.delete(id) + } + resolveAdmissionIdleWaiters(args.usageByPty, args.pressure, args.idleWaiters, id) + } + return pressureBytes +} + +export function takePausedGeneration( + paused: Map, + providerGeneration: number +): SshPtyModelAdmissionKey[] { + const removed: SshPtyModelAdmissionKey[] = [] + for (const [id, key] of paused) { + if (key.providerGeneration === providerGeneration) { + paused.delete(id) + removed.push(key) + } + } + return removed +} + +export function resolveAdmissionIdleWaiters( + usageByPty: ReadonlyMap, + pressure: readonly AdmissionEntry[], + waitersById: Map void>>, + id: string +): void { + const usage = usageByPty.get(id) + const stillPressured = pressure.some((entry) => admissionKeyId(entry.key) === id) + if (usage?.running || (usage?.queued.length ?? 0) > 0 || stillPressured) { + return + } + const waiters = waitersById.get(id) + if (!waiters) { + return + } + waitersById.delete(id) + for (const resolve of waiters) { + resolve() + } +} diff --git a/src/main/ipc/ssh-pty-model-admission-limits.ts b/src/main/ipc/ssh-pty-model-admission-limits.ts new file mode 100644 index 00000000000..865242de284 --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-limits.ts @@ -0,0 +1,38 @@ +import type { SshPtyModelAdmissionOptions } from './ssh-pty-model-admission-contract' + +export type SshPtyModelAdmissionLimits = Readonly<{ + perPtyHighSourceUnits: number + perPtyHighBytes: number + perPtyLowSourceUnits: number + perPtyLowBytes: number + globalHighSourceUnits: number + globalHighBytes: number + globalLowSourceUnits: number + globalLowBytes: number + pressureMaxFrames: number + pressureMaxBytes: number +}> + +const DEFAULT_PER_PTY_HIGH_SOURCE_UNITS = 256 * 1024 +const DEFAULT_PER_PTY_HIGH_BYTES = 2 * 1024 * 1024 +const DEFAULT_GLOBAL_HIGH_SOURCE_UNITS = 50 * DEFAULT_PER_PTY_HIGH_SOURCE_UNITS +const DEFAULT_GLOBAL_HIGH_BYTES = 64 * 1024 * 1024 + +export function resolveSshPtyModelAdmissionLimits( + options: SshPtyModelAdmissionOptions +): SshPtyModelAdmissionLimits { + const perPtyHighSourceUnits = options.perPtyHighSourceUnits ?? DEFAULT_PER_PTY_HIGH_SOURCE_UNITS + const perPtyHighBytes = options.perPtyHighBytes ?? DEFAULT_PER_PTY_HIGH_BYTES + return { + perPtyHighSourceUnits, + perPtyHighBytes, + perPtyLowSourceUnits: options.perPtyLowSourceUnits ?? Math.floor(perPtyHighSourceUnits / 2), + perPtyLowBytes: options.perPtyLowBytes ?? Math.floor(perPtyHighBytes / 2), + globalHighSourceUnits: options.globalHighSourceUnits ?? DEFAULT_GLOBAL_HIGH_SOURCE_UNITS, + globalHighBytes: options.globalHighBytes ?? DEFAULT_GLOBAL_HIGH_BYTES, + globalLowSourceUnits: options.globalLowSourceUnits ?? 8 * 1024 * 1024, + globalLowBytes: options.globalLowBytes ?? 48 * 1024 * 1024, + pressureMaxFrames: options.pressureMaxFrames ?? 64, + pressureMaxBytes: options.pressureMaxBytes ?? 1024 * 1024 + } +} diff --git a/src/main/ipc/ssh-pty-model-admission-migration.ts b/src/main/ipc/ssh-pty-model-admission-migration.ts new file mode 100644 index 00000000000..fc860f4716c --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-migration.ts @@ -0,0 +1,91 @@ +import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract' +import type { AdmissionCharge, AdmissionEntry, PtyUsage } from './ssh-pty-model-admission-entry' +import { admissionError, admissionKeyId } from './ssh-pty-model-admission-entry' +import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure' + +export function beginSshPtyModelAdmissionMigration(args: { + key: SshPtyModelAdmissionKey + migratingPtys: Set + pressure: SshPtyModelAdmissionPressure + usageByPty: Map + release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void + cleanup: (id: string, usage: PtyUsage) => void +}): void { + const id = admissionKeyId(args.key) + if (args.migratingPtys.has(id)) { + return + } + args.migratingPtys.add(id) + const error = admissionError('ssh_model_migration_queued_canceled') + args.pressure.cancelQueuedPty(args.key, error) + const usage = args.usageByPty.get(id) + if (!usage) { + return + } + const queued = usage.queued + usage.queued = [] + for (const entry of queued) { + cancelQueuedEntry(entry, error, args.release) + } + args.cleanup(id, usage) +} + +export function closeSshPtyModelAdmissionMigrations( + migratingPtys: Set, + providerGeneration: number +): void { + const prefix = `${providerGeneration}\0` + for (const id of migratingPtys) { + if (id.startsWith(prefix)) { + migratingPtys.delete(id) + } + } +} + +export function settleSshPtyModelAdmissionFailure(args: { + id: string + usage: PtyUsage + entry: AdmissionEntry + error: Error + migratingPtys: ReadonlySet + closingGenerations: Set + release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void + closeGeneration: (providerGeneration: number) => void + cleanup: (id: string, usage: PtyUsage) => void +}): void { + if (args.entry.state !== 'running' || args.usage.running !== args.entry) { + return + } + const migrationOwnsFailure = args.migratingPtys.has(args.id) + if (!migrationOwnsFailure) { + args.closingGenerations.add(args.entry.key.providerGeneration) + } + args.usage.running = null + args.entry.state = 'settled' + args.release(args.entry.key, args.entry.charge) + args.entry.reject(migrationOwnsFailure ? migrationCompletionError(args.error) : args.error) + if (!migrationOwnsFailure) { + args.closeGeneration(args.entry.key.providerGeneration) + } + args.cleanup(args.id, args.usage) +} + +function migrationCompletionError(cause: Error): Error { + return Object.assign(new Error(cause.message), { + code: 'ssh_model_migration_completion_failed', + cause + }) +} + +function cancelQueuedEntry( + entry: AdmissionEntry, + error: Error, + release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void +): void { + if (entry.state === 'settled') { + return + } + entry.state = 'settled' + release(entry.key, entry.charge) + entry.reject(error) +} diff --git a/src/main/ipc/ssh-pty-model-admission-pressure.ts b/src/main/ipc/ssh-pty-model-admission-pressure.ts new file mode 100644 index 00000000000..256c37bb8dd --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-pressure.ts @@ -0,0 +1,153 @@ +import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract' +import { + admissionKeyId, + pressureHasAdmissionKey, + takePausedGeneration, + type AdmissionEntry, + type PtyUsage +} from './ssh-pty-model-admission-entry' +import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits' + +type PressureOptions = { + limits: SshPtyModelAdmissionLimits + pauseProvider: (key: SshPtyModelAdmissionKey) => boolean + resumeProvider: (key: SshPtyModelAdmissionKey) => void +} + +type PromotionOptions = { + usageByPty: ReadonlyMap + disposed: boolean + canReserve: (entry: AdmissionEntry) => boolean + reserve: (entry: AdmissionEntry) => void + isBelowGlobalLowWatermark: () => boolean +} + +export class SshPtyModelAdmissionPressure { + private readonly entries: AdmissionEntry[] = [] + private readonly pausedKeys = new Map() + private retainedBytes = 0 + + constructor(private readonly options: PressureOptions) {} + + get values(): readonly AdmissionEntry[] { + return this.entries + } + + get frameCount(): number { + return this.entries.length + } + + get bytes(): number { + return this.retainedBytes + } + + get pausedPtyCount(): number { + return this.pausedKeys.size + } + + get pausedProviderGenerations(): ReadonlySet { + return new Set(Array.from(this.pausedKeys.values(), (key) => key.providerGeneration)) + } + + has(key: SshPtyModelAdmissionKey): boolean { + return pressureHasAdmissionKey(this.entries, key) + } + + admit(entry: AdmissionEntry): boolean { + const id = admissionKeyId(entry.key) + const paused = this.pausedKeys.has(id) || this.options.pauseProvider(entry.key) + if (paused) { + this.pausedKeys.set(id, entry.key) + } + if ( + !paused || + this.entries.length >= this.options.limits.pressureMaxFrames || + this.retainedBytes + entry.charge.bytes > this.options.limits.pressureMaxBytes + ) { + return false + } + entry.state = 'pressure' + this.entries.push(entry) + this.retainedBytes += entry.charge.bytes + return true + } + + cancelGeneration( + providerGeneration: number, + cancelPressureAndReserved: (entries: AdmissionEntry[]) => number + ): void { + this.retainedBytes -= cancelPressureAndReserved(this.entries) + for (const key of takePausedGeneration(this.pausedKeys, providerGeneration)) { + this.resume(key) + } + } + + cancelPty(key: SshPtyModelAdmissionKey, error: Error, cancelReserved: () => void): void { + this.cancelQueuedPty(key, error) + cancelReserved() + } + + cancelQueuedPty(key: SshPtyModelAdmissionKey, error: Error): void { + const id = admissionKeyId(key) + const canceled: AdmissionEntry[] = [] + for (let index = this.entries.length - 1; index >= 0; index--) { + if (admissionKeyId(this.entries[index]!.key) === id) { + canceled.push(this.entries.splice(index, 1)[0]!) + } + } + this.rejectPressureEntries(canceled, error) + const paused = this.pausedKeys.get(id) + if (paused) { + this.pausedKeys.delete(id) + this.resume(paused) + } + } + + promoteAndResume(options: PromotionOptions): void { + if (options.disposed) { + return + } + for (let index = 0; index < this.entries.length; ) { + const entry = this.entries[index]! + const hasEarlierEntryForPty = this.entries + .slice(0, index) + .some((earlier) => admissionKeyId(earlier.key) === admissionKeyId(entry.key)) + if (hasEarlierEntryForPty || !options.canReserve(entry)) { + index++ + continue + } + this.entries.splice(index, 1) + this.retainedBytes -= entry.charge.bytes + options.reserve(entry) + } + if (!options.isBelowGlobalLowWatermark()) { + return + } + for (const [id, key] of this.pausedKeys) { + const usage = options.usageByPty.get(id) + if ( + this.has(key) || + (usage?.sourceUnits ?? 0) > this.options.limits.perPtyLowSourceUnits || + (usage?.bytes ?? 0) > this.options.limits.perPtyLowBytes + ) { + continue + } + this.pausedKeys.delete(id) + this.resume(key) + } + } + + private rejectPressureEntries(entries: readonly AdmissionEntry[], error: Error): void { + for (const entry of entries) { + this.retainedBytes -= entry.charge.bytes + entry.state = 'settled' + entry.reject(error) + } + } + + private resume(key: SshPtyModelAdmissionKey): void { + try { + this.options.resumeProvider(key) + } catch {} + } +} diff --git a/src/main/ipc/ssh-pty-model-admission-snapshot.ts b/src/main/ipc/ssh-pty-model-admission-snapshot.ts new file mode 100644 index 00000000000..dfadd32e83c --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission-snapshot.ts @@ -0,0 +1,18 @@ +import type { SshPtyModelAdmissionDebugSnapshot } from './ssh-pty-model-admission-contract' +import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure' + +export function sshPtyModelAdmissionSnapshot( + sourceUnits: number, + bytes: number, + pressure: SshPtyModelAdmissionPressure, + migratingPtys: ReadonlySet +): SshPtyModelAdmissionDebugSnapshot { + return { + sourceUnits, + bytes, + pressureFrames: pressure.frameCount, + pressureBytes: pressure.bytes, + pausedPtys: pressure.pausedPtyCount, + migratingPtys: migratingPtys.size + } +} diff --git a/src/main/ipc/ssh-pty-model-admission.test.ts b/src/main/ipc/ssh-pty-model-admission.test.ts new file mode 100644 index 00000000000..bda8512601d --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it, vi } from 'vitest' +import { SshPtyModelAdmission } from './ssh-pty-model-admission' + +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve + reject = promiseReject + }) + return { promise, resolve, reject } +} + +function accept(admission: SshPtyModelAdmission, completion: Promise) { + return admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'data', 4, () => ({ + sequence: 4, + completion + })) +} + +describe('SshPtyModelAdmission', () => { + it('freezes migration while retaining the running raw completion', async () => { + const runningCompletion = deferred() + const admission = new SshPtyModelAdmission() + const running = accept(admission, runningCompletion.promise) + const queued = accept(admission, Promise.resolve()) + + admission.beginMigration({ ptyId: 'pty-1', providerGeneration: 7 }) + + await expect(queued).rejects.toThrow('ssh_model_migration_queued_canceled') + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 4 }) + await expect( + admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'late', 4, () => ({ + sequence: 8, + completion: Promise.resolve() + })) + ).rejects.toThrow('ssh_model_admission_migrating') + + runningCompletion.resolve() + await expect(running).resolves.toMatchObject({ sequence: 4 }) + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + }) + + it('cancels a never-settling running entry when its generation closes', async () => { + const admission = new SshPtyModelAdmission() + const receipt = accept(admission, new Promise(() => {})) + const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 }) + + admission.closeGeneration(7, 'provider-closed') + + await expect(receipt).rejects.toThrow('provider-closed') + await expect(idle).resolves.toBeUndefined() + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + }) + + it('cancels a never-settling running entry on disposal', async () => { + const admission = new SshPtyModelAdmission() + const receipt = accept(admission, new Promise(() => {})) + const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 }) + + admission.dispose() + + await expect(receipt).rejects.toThrow('ssh_model_admission_disposed') + await expect(idle).resolves.toBeUndefined() + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + }) + + it('keeps non-migrating callback failure generation-fatal across sibling PTYs', async () => { + const admission = new SshPtyModelAdmission() + const failedCompletion = deferred() + const siblingCompletion = deferred() + const failed = accept(admission, failedCompletion.promise) + const sibling = admission.accept({ ptyId: 'pty-2', providerGeneration: 7 }, 'data', 4, () => ({ + sequence: 4, + completion: siblingCompletion.promise + })) + + failedCompletion.reject(new Error('emulator failed')) + + await expect(failed).rejects.toThrow('emulator failed') + await expect(sibling).rejects.toThrow('ssh_model_admission_completion_failed') + await expect( + admission.accept({ ptyId: 'pty-3', providerGeneration: 7 }, 'data', 4, () => ({ + sequence: 4, + completion: Promise.resolve() + })) + ).rejects.toThrow('ssh_model_admission_generation_closed') + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + siblingCompletion.resolve() + await Promise.resolve() + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + }) + + it('resumes every paused provider generation exactly once on disposal', async () => { + const resumeProvider = vi.fn() + const admission = new SshPtyModelAdmission({ + perPtyHighSourceUnits: 4, + perPtyHighBytes: 1024, + globalHighSourceUnits: 4, + globalHighBytes: 1024, + pressureMaxFrames: 1, + pressureMaxBytes: 1024, + pauseProvider: () => true, + resumeProvider + }) + const running = accept(admission, new Promise(() => {})) + const pressured = accept(admission, Promise.resolve()) + const rejected = admission.accept({ ptyId: 'pty-2', providerGeneration: 8 }, 'data', 4, () => ({ + sequence: 4, + completion: Promise.resolve() + })) + + await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted') + admission.dispose() + admission.dispose() + + await expect(running).rejects.toThrow('ssh_model_admission_disposed') + await expect(pressured).rejects.toThrow('ssh_model_admission_disposed') + expect(resumeProvider.mock.calls).toEqual([ + [{ ptyId: 'pty-1', providerGeneration: 7 }], + [{ ptyId: 'pty-2', providerGeneration: 8 }] + ]) + }) + + it.each(['resolve', 'reject'] as const)( + 'ignores a late completion %s after cancellation', + async (settle) => { + const completion = deferred() + const admission = new SshPtyModelAdmission() + const onResolve = vi.fn() + const onReject = vi.fn() + const receipt = accept(admission, completion.promise) + const observed = receipt.then(onResolve, onReject) + + admission.closeGeneration(7, 'provider-closed') + await observed + if (settle === 'resolve') { + completion.resolve() + } else { + completion.reject(new Error('late emulator failure')) + } + await Promise.resolve() + + expect(onResolve).not.toHaveBeenCalled() + expect(onReject).toHaveBeenCalledTimes(1) + expect(onReject.mock.calls[0]?.[0]).toMatchObject({ message: 'provider-closed' }) + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + } + ) +}) diff --git a/src/main/ipc/ssh-pty-model-admission.ts b/src/main/ipc/ssh-pty-model-admission.ts new file mode 100644 index 00000000000..69a9036e2bd --- /dev/null +++ b/src/main/ipc/ssh-pty-model-admission.ts @@ -0,0 +1,315 @@ +import type { + SshPtyModelAdmissionKey, + SshPtyModelAdmissionOptions, + SshPtyModelAdmissionReceipt +} from './ssh-pty-model-admission-contract' +import { + admissionError, + admissionKeyId, + canReserveAdmission, + cancelAdmissionGeneration, + retainedBytes, + resolveAdmissionIdleWaiters, + type AdmissionCharge, + type AdmissionEntry, + type PtyUsage +} from './ssh-pty-model-admission-entry' +import { resolveSshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits' +import * as modelAdmissionMigration from './ssh-pty-model-admission-migration' +import { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure' +import { sshPtyModelAdmissionSnapshot } from './ssh-pty-model-admission-snapshot' + +export type * from './ssh-pty-model-admission-contract' + +export class SshPtyModelAdmission { + private readonly limits: ReturnType + private readonly closeProvider: (providerGeneration: number, reason: string) => void + private readonly usageByPty = new Map() + private readonly pressure: SshPtyModelAdmissionPressure + private readonly idleWaiters = new Map void>>() + private readonly closingGenerations = new Set() + private readonly migratingPtys = new Set() + private globalSourceUnits = 0 + private globalBytes = 0 + private disposed = false + + constructor(options: SshPtyModelAdmissionOptions = {}) { + this.limits = resolveSshPtyModelAdmissionLimits(options) + this.pressure = new SshPtyModelAdmissionPressure({ + limits: this.limits, + pauseProvider: options.pauseProvider ?? (() => false), + resumeProvider: options.resumeProvider ?? (() => {}) + }) + this.closeProvider = options.closeProvider ?? (() => {}) + } + + accept( + key: SshPtyModelAdmissionKey, + data: string, + sourceUnits: number, + run: () => { sequence: number; completion: Promise } + ): Promise { + if (this.disposed) { + return Promise.reject(admissionError('ssh_model_admission_disposed')) + } + if (this.closingGenerations.has(key.providerGeneration)) { + return Promise.reject(admissionError('ssh_model_admission_generation_closed')) + } + if (this.migratingPtys.has(admissionKeyId(key))) { + return Promise.reject(admissionError('ssh_model_admission_migrating')) + } + const charge = { sourceUnits, bytes: retainedBytes(data) } + return new Promise((resolve, reject) => { + const entry: AdmissionEntry = { + key: { ...key }, + charge, + run, + resolve, + reject, + state: 'queued' + } + if (this.canReserve(key, charge) && !this.pressure.has(key)) { + this.reserveAndQueue(entry) + return + } + if (!this.pressure.admit(entry)) { + entry.reject(admissionError('ssh_model_admission_pressure_exhausted')) + this.closeProvider(entry.key.providerGeneration, 'model-admission-pressure') + } + }) + } + + closeGeneration(providerGeneration: number, reason = 'provider-generation-closed'): void { + this.closingGenerations.add(providerGeneration) + const error = admissionError(reason) + this.pressure.cancelGeneration(providerGeneration, (pressure) => + cancelAdmissionGeneration({ + pressure, + usageByPty: this.usageByPty, + idleWaiters: this.idleWaiters, + providerGeneration, + error, + release: (key, charge) => this.release(key, charge) + }) + ) + modelAdmissionMigration.closeSshPtyModelAdmissionMigrations( + this.migratingPtys, + providerGeneration + ) + const generationPrefix = `${providerGeneration}\0` + for (const id of this.idleWaiters.keys()) { + if (id.startsWith(generationPrefix)) { + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) + } + } + } + + beginMigration(key: SshPtyModelAdmissionKey): void { + modelAdmissionMigration.beginSshPtyModelAdmissionMigration({ + key, + migratingPtys: this.migratingPtys, + pressure: this.pressure, + usageByPty: this.usageByPty, + release: (entryKey, charge) => this.release(entryKey, charge), + cleanup: (id, usage) => this.cleanupUsage(id, usage) + }) + const id = admissionKeyId(key) + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) + } + + cancelPty(key: SshPtyModelAdmissionKey, reason: string): void { + const id = admissionKeyId(key) + const error = admissionError(reason) + this.pressure.cancelPty(key, error, () => { + const usage = this.usageByPty.get(id) + if (usage) { + const canceled = [...usage.queued, ...(usage.running ? [usage.running] : [])] + usage.queued = [] + usage.running = null + for (const entry of canceled) { + if (entry.state === 'settled') { + continue + } + entry.state = 'settled' + this.release(entry.key, entry.charge) + entry.reject(error) + } + this.usageByPty.delete(id) + } + }) + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) + } + + whenIdle(key: SshPtyModelAdmissionKey): Promise { + const id = admissionKeyId(key) + const usage = this.usageByPty.get(id) + const hasPressure = this.pressure.has(key) + if ((!usage || (!usage.running && usage.queued.length === 0)) && !hasPressure) { + return Promise.resolve() + } + return new Promise((resolve) => { + const waiters = this.idleWaiters.get(id) ?? new Set<() => void>() + waiters.add(resolve) + this.idleWaiters.set(id, waiters) + }) + } + + dispose(): void { + if (this.disposed) { + return + } + this.disposed = true + const generations = new Set() + for (const usage of this.usageByPty.values()) { + if (usage.running) { + generations.add(usage.running.key.providerGeneration) + } + for (const entry of usage.queued) { + generations.add(entry.key.providerGeneration) + } + } + for (const entry of this.pressure.values) { + generations.add(entry.key.providerGeneration) + } + for (const generation of this.pressure.pausedProviderGenerations) { + generations.add(generation) + } + for (const generation of generations) { + this.closeGeneration(generation, 'ssh_model_admission_disposed') + } + } + + getDebugSnapshot() { + return sshPtyModelAdmissionSnapshot( + this.globalSourceUnits, + this.globalBytes, + this.pressure, + this.migratingPtys + ) + } + + private canReserve(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): boolean { + if (this.migratingPtys.has(admissionKeyId(key))) { + return false + } + return canReserveAdmission({ + key, + charge, + limits: this.limits, + usageByPty: this.usageByPty, + closingGenerations: this.closingGenerations, + globalSourceUnits: this.globalSourceUnits, + globalBytes: this.globalBytes + }) + } + + private reserveAndQueue(entry: AdmissionEntry): void { + const id = admissionKeyId(entry.key) + let usage = this.usageByPty.get(id) + if (!usage) { + usage = { sourceUnits: 0, bytes: 0, queued: [], running: null } + this.usageByPty.set(id, usage) + } + usage.sourceUnits += entry.charge.sourceUnits + usage.bytes += entry.charge.bytes + this.globalSourceUnits += entry.charge.sourceUnits + this.globalBytes += entry.charge.bytes + entry.state = 'queued' + usage.queued.push(entry) + this.startNext(id, usage) + } + + private startNext(id: string, usage: PtyUsage): void { + if (usage.running) { + return + } + const entry = usage.queued.shift() + if (!entry) { + this.promotePressureAndResumeProvider() + return + } + usage.running = entry + entry.state = 'running' + let execution: { sequence: number; completion: Promise } + try { + execution = entry.run() + } catch (error) { + if (this.finishEntry(id, usage, entry)) { + entry.reject(error instanceof Error ? error : new Error(String(error))) + } + return + } + void execution.completion.then( + () => { + if (!this.finishEntry(id, usage, entry)) { + return + } + entry.resolve({ + ptyId: entry.key.ptyId, + providerGeneration: entry.key.providerGeneration, + sequence: execution.sequence + }) + }, + (error) => { + this.failEntry(id, usage, entry, error instanceof Error ? error : new Error(String(error))) + } + ) + } + + private failEntry(id: string, usage: PtyUsage, entry: AdmissionEntry, error: Error): void { + modelAdmissionMigration.settleSshPtyModelAdmissionFailure({ + id, + usage, + entry, + error, + migratingPtys: this.migratingPtys, + closingGenerations: this.closingGenerations, + release: (key, charge) => this.release(key, charge), + closeGeneration: (providerGeneration) => + this.closeGeneration(providerGeneration, 'ssh_model_admission_completion_failed'), + cleanup: (entryId, entryUsage) => this.cleanupUsage(entryId, entryUsage) + }) + } + + private finishEntry(id: string, usage: PtyUsage, entry: AdmissionEntry): boolean { + if (entry.state !== 'running' || usage.running !== entry) { + return false + } + usage.running = null + entry.state = 'settled' + this.release(entry.key, entry.charge) + this.cleanupUsage(id, usage) + return true + } + + private cleanupUsage(id: string, usage: PtyUsage): void { + this.startNext(id, usage) + if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) { + this.usageByPty.delete(id) + } + resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id) + } + + private release(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): void { + const usage = this.usageByPty.get(admissionKeyId(key)) + if (usage) { + usage.sourceUnits = Math.max(0, usage.sourceUnits - charge.sourceUnits) + usage.bytes = Math.max(0, usage.bytes - charge.bytes) + } + this.globalSourceUnits = Math.max(0, this.globalSourceUnits - charge.sourceUnits) + this.globalBytes = Math.max(0, this.globalBytes - charge.bytes) + this.promotePressureAndResumeProvider() + } + + private promotePressureAndResumeProvider(): void { + this.pressure.promoteAndResume({ + usageByPty: this.usageByPty, + disposed: this.disposed, + canReserve: (entry) => this.canReserve(entry.key, entry.charge), + reserve: (entry) => this.reserveAndQueue(entry), + isBelowGlobalLowWatermark: () => + this.globalSourceUnits <= this.limits.globalLowSourceUnits && + this.globalBytes <= this.limits.globalLowBytes + }) + } +} diff --git a/src/main/ipc/ssh-pty-output-exit-deadline.test.ts b/src/main/ipc/ssh-pty-output-exit-deadline.test.ts new file mode 100644 index 00000000000..6dbb81a188c --- /dev/null +++ b/src/main/ipc/ssh-pty-output-exit-deadline.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RemoteTerminalSourceRangeStreamIdentity } from '../runtime/remote-terminal-source-range-consumer' +import { + SshPtyOutputIntake, + type SshPtyOutputIntakeDependencies, + type SshPtyOutputReceipt +} from './ssh-pty-output-intake' +import type { SshPtySourceCancellationProof } from './ssh-pty-output-intake-contract' + +type ExitDeadlineHarness = Readonly<{ + intake: SshPtyOutputIntake + dependencies: SshPtyOutputIntakeDependencies + cancelSourceDelivery: ReturnType + releaseExit: ReturnType + exits: string[] +}> + +const stream: RemoteTerminalSourceRangeStreamIdentity = { + ptyId: 'pty-1', + consumerId: 'remote-1', + streamGeneration: 'stream-1' +} + +function createHarness( + cancelSourceDelivery: NonNullable +): ExitDeadlineHarness { + const exits: string[] = [] + const releaseExit = vi.fn() + const cancellation = vi.fn(cancelSourceDelivery) + const dependencies: SshPtyOutputIntakeDependencies = { + getModelSequence: () => 0, + acceptModel: (event) => ({ sequence: event.rawLength, completion: Promise.resolve() }), + project: vi.fn(), + prepareExit: vi.fn(() => releaseExit), + finalizeExit: () => exits.push('exit'), + pauseProvider: vi.fn(() => true), + resumeProvider: vi.fn(), + closeProvider: vi.fn(), + cancelSourceDelivery: cancellation + } + return { + intake: new SshPtyOutputIntake(dependencies, { + exitBarrierMs: 10, + exitCancellationProofMs: 100 + }), + dependencies, + cancelSourceDelivery: cancellation, + releaseExit, + exits + } +} + +async function publishSource(harness: ExitDeadlineHarness): Promise { + const remote = harness.intake.getRemoteSourceRangeConsumerHooks() + expect(remote.attach(stream)).toBe(true) + const receipt = await harness.intake.acceptData({ + id: 'pty-1', + data: 'aaaa', + providerGeneration: 1, + ptyIncarnation: 'incarnation-1', + rawLength: 4, + transformed: false, + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + harness.intake.publishProjectionPrefix([receipt.projection.identity.projectionSemanticsId], 4, 4) + return receipt +} + +function acceptExit(harness: ExitDeadlineHarness) { + return harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) +} + +describe('SshPtyOutputExitDeadline', () => { + it('transfers published projections before cancellation proof reclaims their spans', async () => { + vi.useFakeTimers() + try { + const harness = createHarness(async () => ({ sentEndSu: 4, creditedEndSu: 0 })) + const receipt = await publishSource(harness) + const exitResult = acceptExit(harness).then( + () => ({ ok: true as const }), + (error: Error) => ({ ok: false as const, error }) + ) + + await vi.advanceTimersByTimeAsync(10) + + expect(await exitResult).toEqual({ ok: true }) + expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce() + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect(harness.dependencies.prepareExit).toHaveBeenCalledOnce() + expect(harness.releaseExit).toHaveBeenCalledOnce() + expect(harness.exits).toEqual(['exit']) + expect(harness.intake.getDebugSnapshot()).toMatchObject({ + projection: { records: 0 }, + source: { openedTokens: 0, ptyIdentities: 0 }, + exitBarriers: 0 + }) + await vi.advanceTimersByTimeAsync(0) + expect(harness.exits).toEqual(['exit']) + expect(() => + harness.intake + .getRemoteSourceRangeConsumerHooks() + .settle(stream, [receipt.projection.desktopSpan!]) + ).not.toThrow() + } finally { + vi.useRealTimers() + } + }) + + it('generation close fences a pending cancellation proof from final exit', async () => { + vi.useFakeTimers() + try { + let resolveCancellation!: (proof: SshPtySourceCancellationProof) => void + const cancellation = new Promise((resolve) => { + resolveCancellation = resolve + }) + const harness = createHarness(() => cancellation) + await publishSource(harness) + const exitResult = acceptExit(harness).then( + () => ({ ok: true as const }), + (error: Error) => ({ ok: false as const, error }) + ) + await vi.advanceTimersByTimeAsync(10) + expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce() + + harness.intake.closeGeneration(1, 'provider-closed') + const closed = await exitResult + expect(closed.ok).toBe(false) + expect(closed.ok ? '' : closed.error.message).toContain('provider-closed') + resolveCancellation({ sentEndSu: 4, creditedEndSu: 0 }) + await vi.advanceTimersByTimeAsync(0) + + expect(harness.exits).toEqual([]) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect(harness.intake.getDebugSnapshot()).toMatchObject({ + projection: { records: 0 }, + source: { openedTokens: 0, ptyIdentities: 0 }, + exitBarriers: 0 + }) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/ipc/ssh-pty-output-exit-deadline.ts b/src/main/ipc/ssh-pty-output-exit-deadline.ts new file mode 100644 index 00000000000..169175f1c44 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-exit-deadline.ts @@ -0,0 +1,239 @@ +import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection' +import type { SshPtyModelAdmission } from './ssh-pty-model-admission' +import type { + SshPtyOutputExitEvent, + SshPtyOutputIntakeDependencies +} from './ssh-pty-output-intake-contract' +import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation' +import type { + SshPtyOutputSourceObligations, + SshPtySourceCancellationProofCommit +} from './ssh-pty-output-source-obligations' + +type SshPtyOutputExitDeadlineDependencies = Readonly<{ + admission: SshPtyModelAdmission + projections: SshPtyLegacyProjectionLedger + sourceObligations: SshPtyOutputSourceObligations + intake: SshPtyOutputIntakeDependencies + barrierMs?: number + cancellationProofMs?: number +}> + +export class SshPtyOutputExitDeadline { + private readonly barriersByGeneration = new Map>() + private readonly preparedExits = new Set() + private readonly preparedExitReleases = new Map void>() + private readonly barrierMs: number + private readonly cancellationProofMs: number + + constructor(private readonly dependencies: SshPtyOutputExitDeadlineDependencies) { + this.barrierMs = dependencies.barrierMs ?? 30_000 + this.cancellationProofMs = dependencies.cancellationProofMs ?? 10_000 + } + + wait( + event: SshPtyOutputExitEvent, + start: (validateNormalExit: () => void) => Promise + ): Promise { + return new Promise((resolve, reject) => { + let timeoutStarted = false + let settled = false + const validateNormalExit = (): void => { + if (timeoutStarted) { + throw outputIntakeError('ssh_exit_delivery_canceled') + } + } + const settle = (result: { ok: true } | { ok: false; error: Error }): void => { + if (settled) { + return + } + settled = true + clearTimeout(barrier.timer) + this.releasePreparedExit(event) + this.remove(event.providerGeneration, barrier) + if (result.ok) { + resolve() + } else { + reject(result.error) + } + } + const barrier: SshPtyExitBarrier = { + timer: setTimeout(() => { + timeoutStarted = true + void this.cancelTimedOutExit(event, barrier).then( + () => settle({ ok: true }), + (error) => { + this.dependencies.intake.closeProvider?.( + event.providerGeneration, + 'ssh-exit-cancellation-proof-failed' + ) + settle({ + ok: false, + error: error instanceof Error ? error : outputIntakeError(String(error)) + }) + } + ) + }, this.barrierMs), + reject: (error) => settle({ ok: false, error }) + } + barrier.timer.unref?.() + let barriers = this.barriersByGeneration.get(event.providerGeneration) + if (!barriers) { + barriers = new Set() + this.barriersByGeneration.set(event.providerGeneration, barriers) + } + barriers.add(barrier) + const promise = start(validateNormalExit) + void promise.then( + () => { + if (!timeoutStarted) { + settle({ ok: true }) + } + }, + (error) => { + if (!timeoutStarted) { + settle({ ok: false, error }) + } + } + ) + }) + } + + closeGeneration(providerGeneration: number, error: Error): void { + const barriers = this.barriersByGeneration.get(providerGeneration) + if (barriers) { + for (const barrier of barriers) { + clearTimeout(barrier.timer) + barrier.reject(error) + } + this.barriersByGeneration.delete(providerGeneration) + } + const prefix = `${providerGeneration}\0` + for (const key of this.preparedExits) { + if (key.startsWith(prefix)) { + this.releasePreparedExitKey(key) + this.preparedExits.delete(key) + } + } + } + + prepareExitOnce(event: SshPtyOutputExitEvent): void { + const key = this.exitKey(event) + if (this.preparedExits.has(key)) { + return + } + const release = this.dependencies.intake.prepareExit(event) + this.preparedExits.add(key) + if (release) { + this.preparedExitReleases.set(key, release) + } + } + + get activeBarriers(): number { + return Array.from(this.barriersByGeneration.values()).reduce( + (total, barriers) => total + barriers.size, + 0 + ) + } + + private async cancelTimedOutExit( + event: SshPtyOutputExitEvent, + barrier: SshPtyExitBarrier + ): Promise { + const cancel = this.dependencies.intake.cancelSourceDelivery + if (!cancel) { + throw outputIntakeError('ssh_source_cancellation_publisher_unavailable') + } + this.dependencies.admission.cancelPty( + { ptyId: event.id, providerGeneration: event.providerGeneration }, + 'ssh_exit_delivery_canceled' + ) + this.dependencies.sourceObligations.sealPty(event) + const cancellation = this.dependencies.sourceObligations.requestPtyCancellationProof( + event, + (request) => cancel(event.providerGeneration, request) + ) + let commit: SshPtySourceCancellationProofCommit | null + try { + commit = await this.withCancellationProofDeadline(cancellation) + } catch (error) { + if (!this.isActive(event.providerGeneration, barrier)) { + return + } + throw error + } + if (!this.isActive(event.providerGeneration, barrier)) { + return + } + if (!commit) { + throw outputIntakeError('ssh_source_cancellation_identity_unavailable') + } + this.dependencies.projections.transferPty(event.id, 'ssh-exit-delivery-canceled') + this.dependencies.sourceObligations.commitPtyCancellationProof(commit) + this.prepareExitOnce(event) + if (!this.isActive(event.providerGeneration, barrier)) { + return + } + this.dependencies.intake.finalizeExit(event) + if (!this.isActive(event.providerGeneration, barrier)) { + return + } + this.dependencies.projections.closePty( + event.id, + event.providerGeneration, + event.ptyIncarnation, + 'ssh-exit-delivery-canceled' + ) + } + + private isActive(providerGeneration: number, barrier: SshPtyExitBarrier): boolean { + return this.barriersByGeneration.get(providerGeneration)?.has(barrier) ?? false + } + + private exitKey(event: SshPtyOutputExitEvent): string { + return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}` + } + + private releasePreparedExit(event: SshPtyOutputExitEvent): void { + const key = this.exitKey(event) + this.preparedExits.delete(key) + this.releasePreparedExitKey(key) + } + + private releasePreparedExitKey(key: string): void { + const release = this.preparedExitReleases.get(key) + if (!release) { + return + } + this.preparedExitReleases.delete(key) + release() + } + + private withCancellationProofDeadline(promise: Promise): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(outputIntakeError('ssh_source_cancellation_proof_timeout')), + this.cancellationProofMs + ) + timer.unref?.() + void promise.then( + (value) => { + clearTimeout(timer) + resolve(value) + }, + (error) => { + clearTimeout(timer) + reject(error) + } + ) + }) + } + + private remove(providerGeneration: number, barrier: SshPtyExitBarrier): void { + const barriers = this.barriersByGeneration.get(providerGeneration) + barriers?.delete(barrier) + if (barriers?.size === 0) { + this.barriersByGeneration.delete(providerGeneration) + } + } +} diff --git a/src/main/ipc/ssh-pty-output-exit.ts b/src/main/ipc/ssh-pty-output-exit.ts new file mode 100644 index 00000000000..99181653135 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-exit.ts @@ -0,0 +1,75 @@ +import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection' +import type { SshPtyModelAdmission } from './ssh-pty-model-admission' +import type { + SshPtyOutputExitEvent, + SshPtyOutputIntakeDependencies +} from './ssh-pty-output-intake-contract' + +export async function settleSshPtyOutputExit(args: { + event: SshPtyOutputExitEvent + admission: SshPtyModelAdmission + projections: SshPtyLegacyProjectionLedger + dependencies: SshPtyOutputIntakeDependencies + validateGeneration: () => void + prepareExit?: () => void + afterAdmissionIdle?: () => void + waitForSourceTerminal?: () => Promise + beforeFinalize?: () => void +}): Promise { + const { + event, + admission, + projections, + dependencies, + validateGeneration, + prepareExit, + afterAdmissionIdle, + waitForSourceTerminal, + beforeFinalize + } = args + await admission.whenIdle({ + ptyId: event.id, + providerGeneration: event.providerGeneration + }) + validateGeneration() + afterAdmissionIdle?.() + try { + if (prepareExit) { + prepareExit() + } else { + dependencies.prepareExit(event) + } + } catch (error) { + projections.closePty( + event.id, + event.providerGeneration, + event.ptyIncarnation, + 'pty-exit-finalize-failed' + ) + dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed') + throw error + } + projections.transferUnpublishedPty( + event.id, + event.providerGeneration, + event.ptyIncarnation, + 'pty-exit-unpublished' + ) + await projections.whenPtyTerminal(event.id, event.providerGeneration, event.ptyIncarnation) + await waitForSourceTerminal?.() + validateGeneration() + try { + beforeFinalize?.() + dependencies.finalizeExit(event) + projections.closePty(event.id, event.providerGeneration, event.ptyIncarnation, 'pty-exit') + } catch (error) { + projections.closePty( + event.id, + event.providerGeneration, + event.ptyIncarnation, + 'pty-exit-finalize-failed' + ) + dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed') + throw error + } +} diff --git a/src/main/ipc/ssh-pty-output-generation-guard.test.ts b/src/main/ipc/ssh-pty-output-generation-guard.test.ts new file mode 100644 index 00000000000..82b40068a01 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-generation-guard.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard' + +const event = (providerGeneration: number) => ({ + id: 'pty-1', + providerGeneration, + ptyIncarnation: 'incarnation-1' +}) + +describe('SshPtyOutputGenerationGuard', () => { + it('compacts sequential closures without weakening stale rejection', () => { + const guard = new SshPtyOutputGenerationGuard(() => false) + for (let generation = 1; generation <= 2_048; generation++) { + guard.closeGeneration(generation) + } + + expect(guard.getDebugSnapshot()).toEqual({ + closedRanges: 1, + activeGaps: 0, + activePtys: 0, + sealedPtys: 0 + }) + expect(() => guard.validate(event(1_024))).toThrow('ssh_output_stale_generation') + expect(() => guard.validate(event(2_049))).not.toThrow() + }) + + it('counts and merges out-of-order live generation gaps exactly', () => { + const guard = new SshPtyOutputGenerationGuard(() => false) + guard.closeGeneration(4) + guard.closeGeneration(2) + + expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 2, activeGaps: 2 }) + guard.closeGeneration(3) + expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 1, activeGaps: 1 }) + expect(() => guard.validate(event(1))).not.toThrow() + expect(() => guard.validate(event(2))).toThrow('ssh_output_stale_generation') + + guard.closeGeneration(1) + expect(guard.getDebugSnapshot()).toEqual({ + closedRanges: 1, + activeGaps: 0, + activePtys: 0, + sealedPtys: 0 + }) + }) +}) diff --git a/src/main/ipc/ssh-pty-output-generation-guard.ts b/src/main/ipc/ssh-pty-output-generation-guard.ts new file mode 100644 index 00000000000..706e40ccd8d --- /dev/null +++ b/src/main/ipc/ssh-pty-output-generation-guard.ts @@ -0,0 +1,107 @@ +import type { SshPtyOutputDataEvent, SshPtyOutputExitEvent } from './ssh-pty-output-intake-contract' +import { + outputIntakeError, + sshPtyGenerationKey, + validOutputLength +} from './ssh-pty-output-intake-validation' +import { SshPtyClosedGenerationRanges } from './ssh-pty-closed-generation-ranges' + +export class SshPtyOutputGenerationGuard { + private readonly latestGenerationByPty = new Map() + private readonly incarnationByPty = new Map() + private readonly sealedPtys = new Set() + private readonly closedGenerations = new SshPtyClosedGenerationRanges() + + constructor(private readonly isDisposed: () => boolean) {} + + validateData(event: SshPtyOutputDataEvent): void { + if ( + !event.id || + !event.ptyIncarnation || + !Number.isSafeInteger(event.providerGeneration) || + event.providerGeneration <= 0 || + !validOutputLength(event.rawLength) || + (event.source !== undefined && + (!event.source.spanId || + !event.source.deliveryToken || + !Number.isSafeInteger(event.source.clientGeneration) || + event.source.clientGeneration <= 0 || + !Number.isSafeInteger(event.source.ownerGeneration) || + event.source.ownerGeneration <= 0 || + !Number.isSafeInteger(event.source.sourceStartSu) || + event.source.sourceStartSu < 0 || + !Number.isSafeInteger(event.source.sourceEndSu) || + event.source.sourceEndSu - event.source.sourceStartSu !== event.rawLength)) + ) { + throw outputIntakeError('ssh_output_invalid_event') + } + this.validate(event) + if (this.sealedPtys.has(sshPtyGenerationKey(event.id, event.providerGeneration))) { + throw outputIntakeError('ssh_output_after_exit') + } + } + + sealExit(event: SshPtyOutputExitEvent): void { + this.validate(event) + const key = sshPtyGenerationKey(event.id, event.providerGeneration) + if (this.sealedPtys.has(key)) { + throw outputIntakeError('ssh_output_duplicate_exit') + } + this.sealedPtys.add(key) + } + + validate(event: { id: string; providerGeneration: number; ptyIncarnation: string }): void { + if (this.isDisposed()) { + throw outputIntakeError('ssh_output_intake_disposed') + } + if (this.closedGenerations.has(event.providerGeneration)) { + throw outputIntakeError('ssh_output_stale_generation') + } + const generation = this.latestGenerationByPty.get(event.id) + if (generation !== undefined && event.providerGeneration < generation) { + throw outputIntakeError('ssh_output_stale_generation') + } + const incarnation = this.incarnationByPty.get(event.id) + if ( + generation === event.providerGeneration && + incarnation !== undefined && + incarnation !== event.ptyIncarnation + ) { + throw outputIntakeError('ssh_output_stale_incarnation') + } + if (generation === undefined || event.providerGeneration > generation) { + this.latestGenerationByPty.set(event.id, event.providerGeneration) + this.incarnationByPty.set(event.id, event.ptyIncarnation) + } + } + + closeGeneration(providerGeneration: number): void { + this.closedGenerations.add(providerGeneration) + for (const [ptyId, generation] of this.latestGenerationByPty) { + if (generation === providerGeneration) { + this.latestGenerationByPty.delete(ptyId) + this.incarnationByPty.delete(ptyId) + this.sealedPtys.delete(sshPtyGenerationKey(ptyId, generation)) + } + } + const prefix = `${providerGeneration}\0` + for (const key of this.sealedPtys) { + if (key.startsWith(prefix)) { + this.sealedPtys.delete(key) + } + } + } + + activeGenerations(): ReadonlySet { + return new Set(this.latestGenerationByPty.values()) + } + + getDebugSnapshot() { + return { + closedRanges: this.closedGenerations.size, + activeGaps: this.closedGenerations.activeGaps, + activePtys: this.latestGenerationByPty.size, + sealedPtys: this.sealedPtys.size + } + } +} diff --git a/src/main/ipc/ssh-pty-output-intake-contract.ts b/src/main/ipc/ssh-pty-output-intake-contract.ts new file mode 100644 index 00000000000..9e2c5d7a101 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-intake-contract.ts @@ -0,0 +1,77 @@ +import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' +import type { + SshPtyModelAdmissionOptions, + SshPtyModelAdmissionReceipt +} from './ssh-pty-model-admission-contract' +import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract' + +export type SshPtySourceCancellationRequest = Readonly<{ + id: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string +}> + +export type SshPtySourceCancellationProof = Readonly<{ + sentEndSu: number + creditedEndSu: number +}> + +export type SshPtyOutputDataEvent = Readonly<{ + id: string + data: string + providerGeneration: number + ptyIncarnation: string + rawLength: number + transformed: boolean + sequence?: number + source?: Readonly<{ + relayPtyId?: string + spanId: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string + sourceStartSu: number + sourceEndSu: number + }> +}> + +export type SshPtyOutputExitEvent = Readonly<{ + id: string + code: number + providerGeneration: number + ptyIncarnation: string +}> + +export type SshPtyOutputReceipt = SshPtyModelAdmissionReceipt & + Readonly<{ projection: LegacySshProjectionSemantics }> + +export type SshPtyOutputIntakeDependencies = { + getModelSequence: (id: string) => number + acceptModel: ( + event: SshPtyOutputDataEvent, + projection: LegacySshProjectionSemantics + ) => { sequence: number; completion: Promise } + project: (event: SshPtyOutputDataEvent, projection: LegacySshProjectionSemantics) => void + prepareExit: (event: SshPtyOutputExitEvent) => void | (() => void) + finalizeExit: (event: SshPtyOutputExitEvent) => void + pauseProvider?: (providerGeneration: number, id: string) => boolean + resumeProvider?: (providerGeneration: number, id: string) => void + closeProvider?: (providerGeneration: number, reason: string) => void + resetModelForMigration?: (providerGeneration: number, id: string) => void + onGenerationClosed?: (providerGeneration: number, reason: string) => void + publishSourceAck?: ( + providerGeneration: number, + batch: PtySourceCreditAckBatch, + onSettled: (result: { ok: true } | { ok: false; error: Error }) => void + ) => void + cancelSourceDelivery?: ( + providerGeneration: number, + request: SshPtySourceCancellationRequest + ) => Promise +} + +export type SshPtyOutputIntakeOptions = SshPtyModelAdmissionOptions & { + exitBarrierMs?: number + exitCancellationProofMs?: number +} diff --git a/src/main/ipc/ssh-pty-output-intake-registry.ts b/src/main/ipc/ssh-pty-output-intake-registry.ts new file mode 100644 index 00000000000..3efdba6113d --- /dev/null +++ b/src/main/ipc/ssh-pty-output-intake-registry.ts @@ -0,0 +1,151 @@ +import type { SshPtyOutputIntake } from './ssh-pty-output-intake' +import type { + SshPtyOutputDataEvent, + SshPtyOutputExitEvent, + SshPtyOutputReceipt, + SshPtySourceCancellationProof, + SshPtySourceCancellationRequest +} from './ssh-pty-output-intake-contract' +import type { SshPtyAcceptedSourceCheckpoint } from './ssh-pty-output-source-obligations' +import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract' +import type { SshPtyOutputGenerationMigration } from './ssh-pty-output-model-migration' + +let installedIntake: SshPtyOutputIntake | null = null +let nextProviderGeneration = 1 +const sourceAckPublishers = new Map() +const sourceCancellationPublishers = new Map() + +type SshPtySourceAckPublisher = ( + batch: PtySourceCreditAckBatch, + onSettled: (result: { ok: true } | { ok: false; error: Error }) => void +) => void + +type SshPtySourceCancellationPublisher = ( + request: SshPtySourceCancellationRequest +) => Promise + +export function allocateSshPtyProviderGeneration(): number { + return nextProviderGeneration++ +} + +export function installSshPtyOutputIntake(intake: SshPtyOutputIntake): () => void { + const previous = installedIntake + installedIntake = intake + previous?.dispose() + return () => { + if (installedIntake === intake) { + installedIntake = null + intake.dispose() + } + } +} + +export function acceptSshPtyOutputData(event: SshPtyOutputDataEvent): Promise { + return installedIntake + ? installedIntake.acceptData(event) + : Promise.reject(outputIntakeUnavailableError()) +} + +export function acceptSshPtyOutputExit(event: SshPtyOutputExitEvent): Promise { + return installedIntake + ? installedIntake.acceptExit(event) + : Promise.reject(outputIntakeUnavailableError()) +} + +export function closeSshPtyOutputGeneration(providerGeneration: number, reason: string): void { + installedIntake?.closeGeneration(providerGeneration, reason) +} + +export function getSshPtyAcceptedSourceCheckpoints( + providerGeneration: number +): readonly SshPtyAcceptedSourceCheckpoint[] { + return installedIntake?.getAcceptedSourceCheckpoints(providerGeneration) ?? [] +} + +export function beginSshPtyOutputGenerationMigration( + providerGeneration: number +): SshPtyOutputGenerationMigration { + return ( + installedIntake?.beginGenerationMigration(providerGeneration) ?? { + byPty: new Map(), + completion: Promise.resolve() + } + ) +} + +export function installSshPtySourceAckPublisher( + providerGeneration: number, + publish: SshPtySourceAckPublisher +): () => void { + if (sourceAckPublishers.has(providerGeneration)) { + throw new Error('ssh_source_ack_publisher_duplicate_generation') + } + sourceAckPublishers.set(providerGeneration, publish) + return () => { + if (sourceAckPublishers.get(providerGeneration) === publish) { + sourceAckPublishers.delete(providerGeneration) + } + } +} + +export function publishSshPtySourceAck( + providerGeneration: number, + batch: PtySourceCreditAckBatch, + onSettled: (result: { ok: true } | { ok: false; error: Error }) => void +): void { + const publisher = sourceAckPublishers.get(providerGeneration) + if (!publisher) { + onSettled({ ok: false, error: new Error('ssh_source_ack_publisher_unavailable') }) + return + } + publisher(batch, onSettled) +} + +export function installSshPtySourceCancellationPublisher( + providerGeneration: number, + cancel: SshPtySourceCancellationPublisher +): () => void { + if (sourceCancellationPublishers.has(providerGeneration)) { + throw new Error('ssh_source_cancellation_publisher_duplicate_generation') + } + sourceCancellationPublishers.set(providerGeneration, cancel) + return () => { + if (sourceCancellationPublishers.get(providerGeneration) === cancel) { + sourceCancellationPublishers.delete(providerGeneration) + } + } +} + +export function cancelSshPtySourceDelivery( + providerGeneration: number, + request: SshPtySourceCancellationRequest +): Promise { + const publisher = sourceCancellationPublishers.get(providerGeneration) + return publisher + ? publisher(request) + : Promise.reject(new Error('ssh_source_cancellation_publisher_unavailable')) +} + +export function applySshPtySourceCancellationProof( + event: SshPtyOutputExitEvent, + proof: SshPtySourceCancellationProof +): boolean { + return installedIntake?.applySourceCancellationProof(event, proof) ?? false +} + +export function applySshPtySourceRecoveryCancellationProof( + event: SshPtyOutputExitEvent, + proof: SshPtySourceCancellationProof +): boolean { + if (!installedIntake) { + return false + } + installedIntake.applySourceRecoveryCancellationProof(event, proof) + return true +} + +function outputIntakeUnavailableError(): Error { + return Object.assign(new Error('ssh_output_intake_unavailable'), { + code: 'ssh_output_intake_unavailable' + }) +} diff --git a/src/main/ipc/ssh-pty-output-intake-test-harness.ts b/src/main/ipc/ssh-pty-output-intake-test-harness.ts new file mode 100644 index 00000000000..9209b897284 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-intake-test-harness.ts @@ -0,0 +1,62 @@ +import { vi } from 'vitest' +import { + SshPtyOutputIntake, + type SshPtyOutputDataEvent, + type SshPtyOutputIntakeDependencies +} from './ssh-pty-output-intake' + +export function sshPtyOutputEvent( + overrides: Partial = {} +): SshPtyOutputDataEvent { + return { + id: 'pty-1', + data: 'aaaa', + providerGeneration: 1, + ptyIncarnation: 'incarnation-1', + rawLength: 4, + transformed: false, + ...overrides + } +} + +export function createSshPtyOutputIntakeHarness( + overrides: Partial = {}, + options: ConstructorParameters[1] = {} +) { + let sequence = 0 + const completions: ReturnType[] = [] + const order: string[] = [] + const dependencies: SshPtyOutputIntakeDependencies = { + getModelSequence: () => sequence, + acceptModel: (input) => { + order.push(`model:${input.data}`) + sequence += input.rawLength + const completion = deferred() + completions.push(completion) + return { sequence, completion: completion.promise } + }, + project: (input) => order.push(`project:${input.data}`), + prepareExit: vi.fn(), + finalizeExit: () => order.push('exit'), + pauseProvider: vi.fn(() => true), + resumeProvider: vi.fn(), + closeProvider: vi.fn(), + ...overrides + } + return { + intake: new SshPtyOutputIntake(dependencies, options), + dependencies, + completions, + order + } +} + +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve + reject = promiseReject + }) + return { promise, resolve, reject } +} diff --git a/src/main/ipc/ssh-pty-output-intake-validation.ts b/src/main/ipc/ssh-pty-output-intake-validation.ts new file mode 100644 index 00000000000..5c9b346860f --- /dev/null +++ b/src/main/ipc/ssh-pty-output-intake-validation.ts @@ -0,0 +1,16 @@ +export type SshPtyExitBarrier = { + timer: ReturnType + reject: (error: Error) => void +} + +export function outputIntakeError(code: string): Error { + return Object.assign(new Error(code), { code }) +} + +export function validOutputLength(value: number): boolean { + return Number.isSafeInteger(value) && value >= 0 +} + +export function sshPtyGenerationKey(ptyId: string, providerGeneration: number): string { + return `${providerGeneration}\0${ptyId}` +} diff --git a/src/main/ipc/ssh-pty-output-intake.test.ts b/src/main/ipc/ssh-pty-output-intake.test.ts new file mode 100644 index 00000000000..be38872da04 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-intake.test.ts @@ -0,0 +1,760 @@ +import { describe, expect, it, vi } from 'vitest' +import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection' +import { + createSshPtyOutputIntakeHarness as createHarness, + sshPtyOutputEvent as event +} from './ssh-pty-output-intake-test-harness' + +describe('SshPtyOutputIntake', () => { + it('plateaus at the model and pressure budgets, then resumes below low water', async () => { + const harness = createHarness( + {}, + { + perPtyHighSourceUnits: 4, + perPtyHighBytes: 1024, + perPtyLowSourceUnits: 1, + perPtyLowBytes: 256, + globalHighSourceUnits: 4, + globalHighBytes: 1024, + globalLowSourceUnits: 1, + globalLowBytes: 256, + pressureMaxFrames: 2, + pressureMaxBytes: 1024 + } + ) + const first = harness.intake.acceptData(event()) + const second = harness.intake.acceptData(event({ data: 'bbbb' })) + const third = harness.intake.acceptData(event({ data: 'cccc' })) + const rejected = harness.intake.acceptData(event({ data: 'dddd' })) + + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 4, + pressureFrames: 2 + }) + await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted') + expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, expect.any(String)) + + harness.completions[0]!.resolve() + await first + harness.completions[1]!.resolve() + await second + harness.completions[2]!.resolve() + await third + + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 0, + bytes: 0, + pressureFrames: 0 + }) + expect(harness.dependencies.resumeProvider).toHaveBeenCalled() + }) + + it('preserves per-PTY FIFO while differently sized pressure entries wait', async () => { + const harness = createHarness( + {}, + { + perPtyHighSourceUnits: 4, + perPtyHighBytes: 4096, + globalHighSourceUnits: 4, + globalHighBytes: 4096, + pressureMaxFrames: 4, + pressureMaxBytes: 4096 + } + ) + const receipts = [ + harness.intake.acceptData(event({ data: 'a', rawLength: 1 })), + harness.intake.acceptData(event({ data: 'bbb', rawLength: 3 })), + harness.intake.acceptData(event({ data: 'cc', rawLength: 2 })), + harness.intake.acceptData(event({ data: 'd', rawLength: 1 })) + ] + + harness.completions[0]!.resolve() + await receipts[0] + harness.completions[1]!.resolve() + await receipts[1] + harness.completions[2]!.resolve() + await receipts[2] + harness.completions[3]!.resolve() + await receipts[3] + + expect(harness.order).toEqual([ + 'model:a', + 'project:a', + 'model:bbb', + 'project:bbb', + 'model:cc', + 'project:cc', + 'model:d', + 'project:d' + ]) + }) + + it('assigns projection sequence ends when queued model capture begins', async () => { + const harness = createHarness( + {}, + { + perPtyHighSourceUnits: 12, + perPtyHighBytes: 4096, + globalHighSourceUnits: 12, + globalHighBytes: 4096 + } + ) + const first = harness.intake.acceptData(event({ data: 'aaaa' })) + const second = harness.intake.acceptData(event({ data: 'bbbb' })) + const third = harness.intake.acceptData(event({ data: 'cccc' })) + + harness.completions[0]!.resolve() + expect((await first).projection.identity.sequenceEnd).toBe(4) + harness.completions[1]!.resolve() + expect((await second).projection.identity.sequenceEnd).toBe(8) + harness.completions[2]!.resolve() + expect((await third).projection.identity.sequenceEnd).toBe(12) + }) + + it('transfers projection state and closes the provider on model failure', async () => { + const harness = createHarness() + const receipt = harness.intake.acceptData(event()) + harness.completions[0]!.reject(new Error('emulator failed')) + + await expect(receipt).rejects.toThrow('emulator failed') + expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1) + expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed') + }) + + it('transfers a committed projection when desktop admission throws', async () => { + const projections: LegacySshProjectionSemantics[] = [] + const harness = createHarness({ + project: (_event, projection) => { + projections.push(projection) + if (projections.length === 1) { + throw new Error('send failed') + } + } + }) + const receipt = harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5 })) + harness.completions[0]!.resolve() + + await expect(receipt).resolves.toMatchObject({ sequence: 5 }) + expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1) + + const next = harness.intake.acceptData(event({ data: '31h', rawLength: 3 })) + harness.completions[1]!.resolve() + await next + expect(projections[1]).toMatchObject({ + identity: { displayStart: 5 }, + beforeScanner: { tail: '\x1b[?20', pendingSubscribe: false }, + decision: 'subscribed' + }) + }) + + it('commits an immutable desktop source identity through projection admission', async () => { + const projections: LegacySshProjectionSemantics[] = [] + const harness = createHarness({ + project: (_event, projection) => projections.push(projection) + }) + const receipt = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 10, + sourceEndSu: 14 + } + }) + ) + harness.completions[0]!.resolve() + + await receipt + expect(projections[0]?.desktopSpan).toMatchObject({ + spanId: 'span-1', + projectionSemanticsId: projections[0]?.identity.projectionSemanticsId, + providerGeneration: 1, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: 10, + sourceEndSu: 14, + displayStart: 0, + displayEnd: 4, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + }) + expect(Object.isFrozen(projections[0]?.desktopSpan)).toBe(true) + }) + + it('exports only the model-settled source boundary before generation close', async () => { + const harness = createHarness() + const receipt = harness.intake.acceptData( + event({ + source: { + spanId: 'recovery-span-1', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-token-1', + sourceStartSu: 10, + sourceEndSu: 14 + } + }) + ) + + expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(10) + harness.completions[0]!.resolve() + await receipt + expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual([ + { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 3, + ownerGeneration: 4, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'delivery-token-1', + acceptedSourceEndSu: 14 + } + ]) + }) + + it('rolls back projection staging when model capture throws synchronously', async () => { + const project = vi.fn() + const harness = createHarness({ + acceptModel: () => { + throw new Error('model reservation failed') + }, + project + }) + + await expect(harness.intake.acceptData(event())).rejects.toThrow('model reservation failed') + expect(project).not.toHaveBeenCalled() + expect(harness.intake.getDebugSnapshot().projection).toMatchObject({ + rolledBack: 1, + records: 0 + }) + expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed') + }) + + it('rolls back projection staging when source reservation validation fails', async () => { + const harness = createHarness() + const source = { + spanId: 'duplicate', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + const first = harness.intake.acceptData(event({ source })) + harness.completions[0]!.resolve() + await first + await expect( + harness.intake.acceptData( + event({ + source: { ...source, sourceStartSu: 4, sourceEndSu: 8 } + }) + ) + ).rejects.toThrow('duplicate') + expect(harness.intake.getDebugSnapshot().projection).toMatchObject({ + rolledBack: 1, + records: 1 + }) + }) + + it('rolls back committed source and scanner facts when model capture throws', async () => { + let attempts = 0 + const harness = createHarness({ + acceptModel: (accepted) => { + attempts++ + if (attempts === 1) { + throw new Error('model reservation failed') + } + return { + sequence: accepted.rawLength, + completion: Promise.resolve() + } + } + }) + const source = { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 5 + } + await expect( + harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5, source })) + ).rejects.toThrow('model reservation failed') + + const retry = await harness.intake.acceptData( + event({ data: '\x1b[?20', rawLength: 5, source: { ...source, spanId: 'span-2' } }) + ) + expect(retry.projection.identity.displayStart).toBe(0) + expect(retry.projection.beforeScanner).toEqual({ tail: '', pendingSubscribe: false }) + }) + + it('rejects stale provider generations without model capture', async () => { + const harness = createHarness() + const current = harness.intake.acceptData(event({ providerGeneration: 2 })) + harness.completions[0]!.resolve() + await current + + await expect(harness.intake.acceptData(event({ providerGeneration: 1 }))).rejects.toThrow( + 'ssh_output_stale_generation' + ) + expect(harness.completions).toHaveLength(1) + }) + + it('keeps exit behind accepted model and projection work', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const dataReceipt = harness.intake.acceptData(event()) + const exitReceipt = harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + await Promise.resolve() + expect(harness.order).toEqual(['model:aaaa', 'project:aaaa']) + + harness.completions[0]!.resolve() + await Promise.all([dataReceipt, exitReceipt]) + expect(harness.order).toEqual(['model:aaaa', 'project:aaaa', 'exit']) + }) + + it('admits queued pre-exit source spans before sealing the token', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const first = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + const second = harness.intake.acceptData( + event({ + data: 'bbbb', + source: { + spanId: 'span-2', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 4, + sourceEndSu: 8 + } + }) + ) + const exit = harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + + harness.completions[0]!.resolve() + await first + harness.completions[1]!.resolve() + await Promise.all([second, exit]) + expect(harness.order).toEqual([ + 'model:aaaa', + 'project:aaaa', + 'model:bbbb', + 'project:bbbb', + 'exit' + ]) + }) + + it('does not pump queued model work after a running completion fails', async () => { + const harness = createHarness() + const first = harness.intake.acceptData(event({ data: 'first' })) + const queued = harness.intake.acceptData(event({ data: 'queued' })) + + harness.completions[0]!.reject(new Error('emulator failed')) + await expect(first).rejects.toThrow('emulator failed') + await expect(queued).rejects.toThrow('ssh_model_admission_completion_failed') + expect(harness.order).toEqual(['model:first', 'project:first']) + expect(harness.completions).toHaveLength(1) + }) + + it('retains exited delivery until published renderer projections settle', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const dataReceipt = harness.intake.acceptData(event()) + harness.completions[0]!.resolve() + const receipt = await dataReceipt + harness.intake.publishProjectionPrefix( + [receipt.projection.identity.projectionSemanticsId], + 4, + 4 + ) + + let exited = false + const exitReceipt = harness.intake + .acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + .then(() => { + exited = true + }) + await Promise.resolve() + + expect(exited).toBe(false) + expect(harness.intake.getDebugSnapshot().projection.records).toBe(1) + expect(harness.intake.settleProjectionPrefix('pty-1', 4)).toBe(4) + await exitReceipt + expect(harness.order.at(-1)).toBe('exit') + expect(harness.intake.getDebugSnapshot().projection.records).toBe(0) + }) + + it('owns renderer exit preparation through finalization and duplicate rejection', async () => { + const releaseRendererExit = vi.fn() + const harness = createHarness({ + prepareExit: vi.fn(() => releaseRendererExit) + }) + const dataReceipt = harness.intake.acceptData(event()) + harness.completions[0]!.resolve() + const receipt = await dataReceipt + harness.intake.publishProjectionPrefix( + [receipt.projection.identity.projectionSemanticsId], + 4, + 4 + ) + const exitEvent = { + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + } + const exit = harness.intake.acceptExit(exitEvent) + await Promise.resolve() + + expect(releaseRendererExit).not.toHaveBeenCalled() + await expect(harness.intake.acceptExit(exitEvent)).rejects.toThrow('ssh_output_duplicate_exit') + expect(releaseRendererExit).not.toHaveBeenCalled() + + harness.intake.settleProjectionPrefix('pty-1', 4) + await exit + expect(releaseRendererExit).toHaveBeenCalledOnce() + }) + + it('releases renderer exit preparation when generation close aborts finalization', async () => { + const releaseRendererExit = vi.fn() + const harness = createHarness({ + prepareExit: vi.fn(() => releaseRendererExit) + }) + const dataReceipt = harness.intake.acceptData(event()) + harness.completions[0]!.resolve() + const receipt = await dataReceipt + harness.intake.publishProjectionPrefix( + [receipt.projection.identity.projectionSemanticsId], + 4, + 4 + ) + const exit = harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + await Promise.resolve() + + harness.intake.closeGeneration(1, 'provider-replaced') + + await expect(exit).rejects.toThrow('provider-replaced') + expect(releaseRendererExit).toHaveBeenCalledOnce() + expect(harness.order).not.toContain('exit') + }) + + it('retains exit until a required remote source consumer settles', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const remote = harness.intake.getRemoteSourceRangeConsumerHooks() + const stream = { + ptyId: 'pty-1', + consumerId: 'remote-1', + streamGeneration: 'stream-1' + } + expect(remote.attach(stream)).toBe(true) + const dataReceipt = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + const receipt = await dataReceipt + + let exited = false + const exitReceipt = harness.intake + .acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + .then(() => { + exited = true + }) + await Promise.resolve() + expect(exited).toBe(false) + + remote.settle(stream, [receipt.projection.desktopSpan!]) + await exitReceipt + expect(harness.order.at(-1)).toBe('exit') + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + }) + + it('keeps timed-out exit projections until generation-close proof', async () => { + const harness = createHarness( + { + cancelSourceDelivery: () => Promise.reject(new Error('cancel transport failed')) + }, + { exitBarrierMs: 1, exitCancellationProofMs: 10 } + ) + const dataReceipt = harness.intake.acceptData(event()) + harness.completions[0]!.resolve() + const receipt = await dataReceipt + harness.intake.publishProjectionPrefix( + [receipt.projection.identity.projectionSemanticsId], + 4, + 4 + ) + + await expect( + harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + ).rejects.toThrow('ssh_source_cancellation_identity_unavailable') + expect(harness.dependencies.closeProvider).toHaveBeenCalledWith( + 1, + 'ssh-exit-cancellation-proof-failed' + ) + expect(harness.intake.getDebugSnapshot().projection.records).toBe(1) + + harness.intake.closeGeneration(1, 'provider-closed') + expect(harness.intake.getDebugSnapshot().projection.records).toBe(0) + }) + + it('cancels only the timed-out source delivery and keeps the provider usable', async () => { + const cancelSourceDelivery = vi.fn(async () => ({ sentEndSu: 4, creditedEndSu: 0 })) + const harness = createHarness( + { cancelSourceDelivery }, + { exitBarrierMs: 1, exitCancellationProofMs: 100 } + ) + const remote = harness.intake.getRemoteSourceRangeConsumerHooks() + const stream = { + ptyId: 'pty-1', + consumerId: 'remote-1', + streamGeneration: 'stream-1' + } + remote.attach(stream) + const dataReceipt = harness.intake.acceptData( + event({ + source: { + spanId: 'span-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + await dataReceipt + + await harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + expect(cancelSourceDelivery).toHaveBeenCalledWith(1, { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect(harness.order.at(-1)).toBe('exit') + + const sibling = harness.intake.acceptData( + event({ id: 'pty-2', ptyIncarnation: 'incarnation-2' }) + ) + harness.completions[1]!.resolve() + await expect(sibling).resolves.toMatchObject({ ptyId: 'pty-2' }) + }) + + it('accepts recovery cancellation proof before any replacement span is admitted', () => { + const harness = createHarness() + + expect(() => + harness.intake.applySourceRecoveryCancellationProof( + { + id: 'pty-1', + code: -1, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }, + { sentEndSu: 8, creditedEndSu: 4 } + ) + ).not.toThrow() + expect(harness.intake.getDebugSnapshot().source).toEqual({ + openedTokens: 0, + ptyIdentities: 0 + }) + }) + + it('reclaims a partially admitted recovery prefix from authoritative proof', async () => { + const harness = createHarness() + const receipt = harness.intake.acceptData( + event({ + source: { + spanId: 'recovery-span', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'recovery-token', + sourceStartSu: 4, + sourceEndSu: 8 + } + }) + ) + harness.completions[0]!.resolve() + await receipt + + harness.intake.applySourceRecoveryCancellationProof( + { + id: 'pty-1', + code: -1, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }, + { sentEndSu: 12, creditedEndSu: 4 } + ) + + expect(harness.intake.getDebugSnapshot().source).toEqual({ + openedTokens: 0, + ptyIdentities: 0 + }) + }) + + it('rejects late same-generation data after ordered exit cleanup', async () => { + const harness = createHarness({}, { exitBarrierMs: 1000 }) + const first = harness.intake.acceptData(event()) + harness.completions[0]!.resolve() + await first + await harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + + await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_after_exit') + await expect( + harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + ).rejects.toThrow('ssh_output_duplicate_exit') + + const next = harness.intake.acceptData( + event({ providerGeneration: 2, ptyIncarnation: 'incarnation-2' }) + ) + harness.completions[1]!.resolve() + await expect(next).resolves.toMatchObject({ + projection: { identity: { ptyIncarnation: 'incarnation-2', displayStart: 0 } } + }) + }) + + it('closes the provider when exit finalization fails', async () => { + const releaseRendererExit = vi.fn() + const harness = createHarness({ + prepareExit: () => releaseRendererExit, + finalizeExit: () => { + throw new Error('renderer exit send failed') + } + }) + + await expect( + harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + ).rejects.toThrow('renderer exit send failed') + expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'pty-exit-finalize-failed') + expect(releaseRendererExit).toHaveBeenCalledOnce() + }) + + it('cancels queued work and exit waiters on generation cleanup', async () => { + const harness = createHarness( + {}, + { + perPtyHighSourceUnits: 8, + perPtyHighBytes: 2048, + globalHighSourceUnits: 8, + globalHighBytes: 2048 + } + ) + const running = harness.intake.acceptData(event()) + const queued = harness.intake.acceptData(event({ data: 'bbbb' })) + const exit = harness.intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 1, + ptyIncarnation: 'incarnation-1' + }) + harness.intake.closeGeneration(1, 'provider-closed') + + await expect(queued).rejects.toThrow('provider-closed') + await expect(exit).rejects.toThrow('provider-closed') + await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_stale_generation') + await expect(running).rejects.toThrow('provider-closed') + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ sourceUnits: 0, bytes: 0 }) + harness.completions[0]!.resolve() + await Promise.resolve() + expect(harness.intake.getDebugSnapshot().exitBarriers).toBe(0) + }) + + it('resumes a paused provider during generation cleanup', async () => { + const harness = createHarness( + {}, + { + perPtyHighSourceUnits: 4, + perPtyHighBytes: 1024, + globalHighSourceUnits: 4, + globalHighBytes: 1024, + pressureMaxFrames: 2, + pressureMaxBytes: 1024 + } + ) + const running = harness.intake.acceptData(event()) + const pressured = harness.intake.acceptData(event({ data: 'bbbb' })) + + harness.intake.closeGeneration(1, 'provider-closed') + + await expect(pressured).rejects.toThrow('provider-closed') + expect(harness.dependencies.resumeProvider).toHaveBeenCalledWith(1, 'pty-1') + await expect(running).rejects.toThrow('provider-closed') + harness.completions[0]!.resolve() + await Promise.resolve() + }) +}) diff --git a/src/main/ipc/ssh-pty-output-intake.ts b/src/main/ipc/ssh-pty-output-intake.ts new file mode 100644 index 00000000000..958713db334 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-intake.ts @@ -0,0 +1,296 @@ +import { + SshPtyLegacyProjectionLedger, + type LegacySshProjectionSemantics +} from './ssh-pty-legacy-projection' +import { SshPtyModelAdmission } from './ssh-pty-model-admission' +import { SshPtyOutputExitDeadline } from './ssh-pty-output-exit-deadline' +import { settleSshPtyOutputExit } from './ssh-pty-output-exit' +import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard' +import { + SshPtyOutputModelMigration, + type SshPtyOutputGenerationMigration, + type SshPtyTrackedModelAdmission +} from './ssh-pty-output-model-migration' +import { + SshPtyOutputSourceObligations, + type SshPtyOutputSourceReservation +} from './ssh-pty-output-source-obligations' +import type { + SshPtyOutputDataEvent, + SshPtyOutputExitEvent, + SshPtyOutputIntakeDependencies, + SshPtyOutputIntakeOptions, + SshPtyOutputReceipt +} from './ssh-pty-output-intake-contract' +import { outputIntakeError } from './ssh-pty-output-intake-validation' + +export type { + SshPtyOutputDataEvent, + SshPtyOutputExitEvent, + SshPtyOutputIntakeDependencies, + SshPtyOutputIntakeOptions, + SshPtyOutputReceipt +} from './ssh-pty-output-intake-contract' + +export class SshPtyOutputIntake { + private readonly projections: SshPtyLegacyProjectionLedger + private readonly sourceObligations: SshPtyOutputSourceObligations + private readonly generationGuard = new SshPtyOutputGenerationGuard(() => this.disposed) + private readonly admission: SshPtyModelAdmission + private readonly modelMigration: SshPtyOutputModelMigration + private readonly exitDeadline: SshPtyOutputExitDeadline + private disposed = false + + constructor( + private readonly dependencies: SshPtyOutputIntakeDependencies, + options: SshPtyOutputIntakeOptions = {} + ) { + this.sourceObligations = new SshPtyOutputSourceObligations(dependencies.publishSourceAck) + this.projections = new SshPtyLegacyProjectionLedger({ + onSettled: (span) => this.sourceObligations.settleDesktop(span, 'renderer-parse'), + onTransferred: (span, reason) => this.sourceObligations.transferDesktop(span, reason) + }) + this.admission = new SshPtyModelAdmission({ + ...options, + pauseProvider: (key) => + this.dependencies.pauseProvider?.(key.providerGeneration, key.ptyId) ?? false, + resumeProvider: (key) => + this.dependencies.resumeProvider?.(key.providerGeneration, key.ptyId), + closeProvider: (providerGeneration, reason) => + this.dependencies.closeProvider?.(providerGeneration, reason) + }) + this.modelMigration = new SshPtyOutputModelMigration( + this.admission, + this.sourceObligations, + (providerGeneration, ptyId) => + this.dependencies.resetModelForMigration?.(providerGeneration, ptyId) + ) + this.exitDeadline = new SshPtyOutputExitDeadline({ + admission: this.admission, + projections: this.projections, + sourceObligations: this.sourceObligations, + intake: this.dependencies, + barrierMs: options.exitBarrierMs, + cancellationProofMs: options.exitCancellationProofMs + }) + } + + acceptData(event: SshPtyOutputDataEvent): Promise { + try { + this.generationGuard.validateData(event) + } catch (error) { + return Promise.reject(error) + } + let projection: LegacySshProjectionSemantics | undefined + let sourceReservation: SshPtyOutputSourceReservation | undefined + const key = { ptyId: event.id, providerGeneration: event.providerGeneration } + const tracked: SshPtyTrackedModelAdmission = { key, started: false } + const receipt = this.admission.accept(key, event.data, event.rawLength, () => { + tracked.started = true + const expectedSequence = this.dependencies.getModelSequence(event.id) + event.rawLength + const reservation = this.projections.reserve({ + ptyId: event.id, + providerGeneration: event.providerGeneration, + ptyIncarnation: event.ptyIncarnation, + data: event.data, + sequenceEnd: expectedSequence, + rawLength: event.rawLength, + transformed: event.transformed, + source: event.source + }) + try { + if (reservation.semantics.desktopSpan) { + sourceReservation = this.sourceObligations.reserve( + event, + reservation.semantics.desktopSpan + ) + } + } catch (error) { + this.projections.rollback(reservation) + throw error + } + try { + projection = this.projections.commit(reservation) + if (sourceReservation) { + this.sourceObligations.commit( + sourceReservation, + event.id, + projection.identity.sequenceEnd + ) + } + } catch (error) { + if (sourceReservation) { + this.sourceObligations.rollback(sourceReservation) + } + if (!this.projections.rollbackCommitted(reservation)) { + this.projections.rollback(reservation) + } + throw error + } + let model: { sequence: number; completion: Promise } + try { + model = this.dependencies.acceptModel(event, projection) + } catch (error) { + if (sourceReservation) { + this.sourceObligations.rollback(sourceReservation) + } + this.projections.rollbackCommitted(reservation) + throw error + } + try { + this.dependencies.project(event, projection) + } catch { + const id = projection.identity.projectionSemanticsId + this.projections.transfer([id], 'projection-admission-failed') + } + return model + }) + const completion = receipt.then( + (modelReceipt) => { + if (!projection) { + throw outputIntakeError('ssh_projection_receipt_missing') + } + if (sourceReservation) { + this.sourceObligations.settleModel(sourceReservation.span) + } + return Object.freeze({ ...modelReceipt, projection }) + }, + (error) => { + if (projection) { + this.projections.transfer( + [projection.identity.projectionSemanticsId], + 'model-admission-failed' + ) + } + const code = (error as { code?: unknown }).code + if ( + code !== 'ssh_exit_delivery_canceled' && + !(typeof code === 'string' && code.startsWith('ssh_model_migration_')) + ) { + this.dependencies.closeProvider?.(event.providerGeneration, 'model-admission-failed') + } + throw error + } + ) + tracked.completion = completion + this.modelMigration.track(tracked) + return completion + } + + async acceptExit(event: SshPtyOutputExitEvent): Promise { + this.generationGuard.sealExit(event) + await this.exitDeadline.wait(event, (validateNormalExit) => + this.finishExit(event, validateNormalExit) + ) + } + + private async finishExit( + event: SshPtyOutputExitEvent, + validateNormalExit: () => void + ): Promise { + await settleSshPtyOutputExit({ + event, + admission: this.admission, + projections: this.projections, + dependencies: this.dependencies, + validateGeneration: () => { + this.generationGuard.validate(event) + validateNormalExit() + }, + prepareExit: () => this.exitDeadline.prepareExitOnce(event), + afterAdmissionIdle: () => this.sourceObligations.sealPty(event), + waitForSourceTerminal: () => this.sourceObligations.whenPtyTerminal(event), + beforeFinalize: () => this.sourceObligations.markExitPublished(event) + }) + } + + publishProjectionPrefix( + ids: readonly string[], + displayChars: number, + accountingChars: number + ): void { + this.projections.publishPrefix(ids, displayChars, accountingChars) + } + + settleProjectionPrefix(ptyId: string, accountingChars: number): number { + return this.projections.settlePublishedPrefix(ptyId, accountingChars) + } + + transferProjections(ids: readonly string[], reason: string): number { + return this.projections.transfer(ids, reason) + } + + transferPtyProjections(ptyId: string, reason: string): number { + return this.projections.transferPty(ptyId, reason) + } + + hasProjectionFromGeneration(ids: readonly string[], providerGeneration: number): boolean { + return ids.some( + (id) => this.projections.get(id)?.identity.providerGeneration === providerGeneration + ) + } + + hasUnpublishedProjection(id: string): boolean { + return this.projections.hasUnpublished(id) + } + + closeGeneration(providerGeneration: number, reason: string): void { + this.generationGuard.closeGeneration(providerGeneration) + this.admission.closeGeneration(providerGeneration, reason) + this.dependencies.onGenerationClosed?.(providerGeneration, reason) + this.projections.closeGeneration(providerGeneration, reason) + this.sourceObligations.closeGeneration(providerGeneration, reason) + this.exitDeadline.closeGeneration(providerGeneration, outputIntakeError(reason)) + } + + dispose(): void { + if (this.disposed) { + return + } + this.disposed = true + for (const generation of this.generationGuard.activeGenerations()) { + this.closeGeneration(generation, 'ssh_output_intake_disposed') + } + this.admission.dispose() + this.sourceObligations.dispose() + } + + getRemoteSourceRangeConsumerHooks() { + return this.sourceObligations.remoteHooks + } + + getAcceptedSourceCheckpoints(providerGeneration: number) { + return this.sourceObligations.acceptedCheckpoints(providerGeneration) + } + + beginGenerationMigration( + providerGeneration: number, + timeoutMs?: number + ): SshPtyOutputGenerationMigration { + return this.modelMigration.beginGeneration(providerGeneration, timeoutMs) + } + + applySourceCancellationProof( + event: SshPtyOutputExitEvent, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): boolean { + return this.sourceObligations.applyCancellationProof(event, proof) + } + + applySourceRecoveryCancellationProof( + event: SshPtyOutputExitEvent, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): void { + this.sourceObligations.applyRecoveryCancellationProof(event, proof) + } + + getDebugSnapshot() { + return { + model: this.admission.getDebugSnapshot(), + projection: this.projections.getDebugSnapshot(), + source: this.sourceObligations.getDebugSnapshot(), + generation: this.generationGuard.getDebugSnapshot(), + exitBarriers: this.exitDeadline.activeBarriers + } + } +} diff --git a/src/main/ipc/ssh-pty-output-model-migration.test.ts b/src/main/ipc/ssh-pty-output-model-migration.test.ts new file mode 100644 index 00000000000..e8007a0c274 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-model-migration.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it, vi } from 'vitest' +import { + createSshPtyOutputIntakeHarness as createHarness, + sshPtyOutputEvent as event +} from './ssh-pty-output-intake-test-harness' + +describe('SshPtyOutputModelMigration', () => { + it('fences a running source span before exporting its migration checkpoint', async () => { + const harness = createHarness() + const first = harness.intake.acceptData( + event({ + data: 'aaaa', + source: { + spanId: 'span-a', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + await first + const second = harness.intake.acceptData( + event({ + data: 'bbbb', + source: { + spanId: 'span-b', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-token-1', + sourceStartSu: 4, + sourceEndSu: 8 + } + }) + ) + + const migration = harness.intake.beginGenerationMigration(1) + const result = migration.byPty.get('pty-1') + expect(result).toBeDefined() + await expect(Promise.race([result, Promise.resolve('pending')])).resolves.toBe('pending') + expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(4) + + harness.completions[1]!.resolve() + await expect(second).resolves.toMatchObject({ sequence: 8 }) + await expect(result).resolves.toMatchObject({ + status: 'settled', + checkpoint: { acceptedSourceEndSu: 8 } + }) + expect(harness.order.filter((entry) => entry === 'project:bbbb')).toHaveLength(1) + await migration.completion + }) + + it('times out one migration, resets its model, and releases retained admission once', async () => { + vi.useFakeTimers() + try { + const resetModelForMigration = vi.fn() + const harness = createHarness({ resetModelForMigration }) + const receipt = harness.intake.acceptData( + event({ + source: { + spanId: 'span-b', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + const migration = harness.intake.beginGenerationMigration(1, 10_000) + const result = migration.byPty.get('pty-1') + + await vi.advanceTimersByTimeAsync(10_000) + + await expect(result).resolves.toMatchObject({ status: 'checkpoint-unavailable' }) + await expect(receipt).rejects.toThrow('ssh_model_migration_timeout') + expect(resetModelForMigration).toHaveBeenCalledOnce() + expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-1') + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 0, + bytes: 0, + migratingPtys: 1 + }) + expect(vi.getTimerCount()).toBe(0) + harness.completions[0]!.resolve() + await Promise.resolve() + expect(resetModelForMigration).toHaveBeenCalledOnce() + harness.intake.closeGeneration(1, 'connection_lost') + expect(harness.intake.getDebugSnapshot().model.migratingPtys).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('contains a running callback failure to its migrating PTY', async () => { + vi.useFakeTimers() + try { + const resetModelForMigration = vi.fn() + const harness = createHarness({ resetModelForMigration }) + const sibling = harness.intake.acceptData( + event({ + id: 'pty-sibling', + ptyIncarnation: 'incarnation-sibling', + source: { + spanId: 'span-sibling', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-sibling', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + await sibling + const failed = harness.intake.acceptData( + event({ + id: 'pty-failed', + ptyIncarnation: 'incarnation-failed', + source: { + spanId: 'span-failed', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-failed', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + const migration = harness.intake.beginGenerationMigration(1) + + harness.completions[1]!.reject(new Error('emulator failed')) + + await expect(failed).rejects.toMatchObject({ + message: 'emulator failed', + code: 'ssh_model_migration_completion_failed' + }) + await expect(migration.byPty.get('pty-failed')).resolves.toEqual({ + status: 'checkpoint-unavailable', + reason: 'completion-failed' + }) + await expect(migration.byPty.get('pty-sibling')).resolves.toMatchObject({ + status: 'settled', + checkpoint: { id: 'pty-sibling', acceptedSourceEndSu: 4 } + }) + await migration.completion + expect(resetModelForMigration).toHaveBeenCalledTimes(1) + expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-failed') + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: 'pty-failed', acceptedSourceEndSu: 0 }), + expect.objectContaining({ id: 'pty-sibling', acceptedSourceEndSu: 4 }) + ]) + ) + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 0, + bytes: 0, + pressureFrames: 0, + migratingPtys: 2 + }) + expect(vi.getTimerCount()).toBe(0) + + harness.completions[1]!.resolve() + await Promise.resolve() + expect(resetModelForMigration).toHaveBeenCalledTimes(1) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect( + harness.intake + .getAcceptedSourceCheckpoints(1) + .find((checkpoint) => checkpoint.id === 'pty-failed')?.acceptedSourceEndSu + ).toBe(0) + + const unrelated = harness.intake.acceptData( + event({ + id: 'pty-unrelated', + providerGeneration: 2, + ptyIncarnation: 'incarnation-unrelated' + }) + ) + harness.completions[2]!.resolve() + await expect(unrelated).resolves.toMatchObject({ providerGeneration: 2 }) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + harness.intake.closeGeneration(1, 'connection_lost') + harness.intake.closeGeneration(2, 'connection_lost') + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 0, + bytes: 0, + migratingPtys: 0 + }) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/ipc/ssh-pty-output-model-migration.ts b/src/main/ipc/ssh-pty-output-model-migration.ts new file mode 100644 index 00000000000..4478e8d0752 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-model-migration.ts @@ -0,0 +1,134 @@ +import type { + SshPtyAcceptedSourceCheckpoint, + SshPtyOutputSourceObligations +} from './ssh-pty-output-source-obligations' +import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract' +import type { SshPtyModelAdmission } from './ssh-pty-model-admission' + +export const SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS = 10_000 + +export type SshPtyOutputMigrationResult = + | Readonly<{ status: 'settled'; checkpoint: SshPtyAcceptedSourceCheckpoint }> + | Readonly<{ status: 'checkpoint-unavailable'; reason: 'completion-failed' | 'timeout' }> + +export type SshPtyOutputGenerationMigration = Readonly<{ + byPty: ReadonlyMap> + completion: Promise +}> + +export type SshPtyTrackedModelAdmission = { + readonly key: SshPtyModelAdmissionKey + started: boolean + completion?: Promise +} + +export class SshPtyOutputModelMigration { + private readonly pendingByPty = new Map>() + + constructor( + private readonly admission: SshPtyModelAdmission, + private readonly sourceObligations: SshPtyOutputSourceObligations, + private readonly resetModel: (providerGeneration: number, ptyId: string) => void + ) {} + + track(record: SshPtyTrackedModelAdmission): void { + const id = migrationKey(record.key) + const records = this.pendingByPty.get(id) ?? new Set() + records.add(record) + this.pendingByPty.set(id, records) + record.completion?.then( + () => this.remove(id, record), + () => this.remove(id, record) + ) + } + + beginGeneration( + providerGeneration: number, + timeoutMs = SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS + ): SshPtyOutputGenerationMigration { + const checkpoints = this.sourceObligations.acceptedCheckpoints(providerGeneration) + const keys = checkpoints.map((checkpoint) => ({ + ptyId: checkpoint.id, + providerGeneration + })) + for (const key of keys) { + this.admission.beginMigration(key) + } + const byPty = new Map>() + for (const key of keys) { + byPty.set(key.ptyId, this.settlePty(key, timeoutMs)) + } + const completion = Promise.allSettled(byPty.values()).then(() => {}) + return Object.freeze({ byPty, completion }) + } + + private async settlePty( + key: SshPtyModelAdmissionKey, + timeoutMs: number + ): Promise { + const running = Array.from(this.pendingByPty.get(migrationKey(key)) ?? []).find( + (record) => record.started + ) + if (!running?.completion) { + return this.settledCheckpoint(key) + } + let timer: ReturnType | undefined + const timeout = new Promise<'timeout'>((resolve) => { + timer = setTimeout(() => resolve('timeout'), normalizedTimeout(timeoutMs)) + timer.unref?.() + }) + try { + const outcome = await Promise.race([ + running.completion.then( + () => 'settled' as const, + () => 'failed' as const + ), + timeout + ]) + if (outcome === 'settled') { + return this.settledCheckpoint(key) + } + const reason = outcome === 'timeout' ? 'timeout' : 'completion-failed' + this.resetModel(key.providerGeneration, key.ptyId) + this.admission.cancelPty(key, `ssh_model_migration_${reason}`) + if (outcome === 'timeout') { + await running.completion.catch(() => {}) + } + return Object.freeze({ status: 'checkpoint-unavailable', reason }) + } finally { + if (timer) { + clearTimeout(timer) + } + } + } + + private settledCheckpoint(key: SshPtyModelAdmissionKey): SshPtyOutputMigrationResult { + const checkpoint = this.sourceObligations.acceptedCheckpoint(key) + if (!checkpoint) { + this.resetModel(key.providerGeneration, key.ptyId) + return Object.freeze({ + status: 'checkpoint-unavailable', + reason: 'completion-failed' + }) + } + return Object.freeze({ status: 'settled', checkpoint }) + } + + private remove(id: string, record: SshPtyTrackedModelAdmission): void { + const records = this.pendingByPty.get(id) + records?.delete(record) + if (records?.size === 0) { + this.pendingByPty.delete(id) + } + } +} + +function migrationKey(key: SshPtyModelAdmissionKey): string { + return `${key.providerGeneration}\0${key.ptyId}` +} + +function normalizedTimeout(timeoutMs: number): number { + return Number.isFinite(timeoutMs) && timeoutMs >= 0 + ? Math.floor(timeoutMs) + : SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS +} diff --git a/src/main/ipc/ssh-pty-output-source-obligations.ts b/src/main/ipc/ssh-pty-output-source-obligations.ts new file mode 100644 index 00000000000..28f13dc2d73 --- /dev/null +++ b/src/main/ipc/ssh-pty-output-source-obligations.ts @@ -0,0 +1,297 @@ +import { + ptySourceDeliveryKey, + samePtySourceDelivery, + type PtySourceDeliveryIdentity, + type PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import type { RemoteTerminalSourceRangeConsumerHooks } from '../runtime/remote-terminal-source-range-consumer' +import type { DesktopProjectionSpan } from './ssh-pty-legacy-projection' +import type { + SshPtyOutputDataEvent, + SshPtyOutputExitEvent, + SshPtyOutputIntakeDependencies, + SshPtySourceCancellationProof, + SshPtySourceCancellationRequest +} from './ssh-pty-output-intake-contract' +import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers' +import type { SshPtySourceAdmissionReservation } from './ssh-pty-source-obligation-contract' +import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' + +export type SshPtyOutputSourceReservation = Readonly<{ + admission: SshPtySourceAdmissionReservation + span: PtySourceSpan +}> + +export type SshPtySourceCancellationProofCommit = Readonly<{ + identity: PtySourceDeliveryIdentity + proof: SshPtySourceCancellationProof +}> + +export type SshPtyAcceptedSourceCheckpoint = Readonly<{ + id: string + providerGeneration: number + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string + acceptedSourceEndSu: number +}> + +export class SshPtyOutputSourceObligations { + private readonly coordinator: SshPtySourceObligationCoordinator + private readonly remoteConsumers: SshPtyRemoteSourceRangeConsumers + private readonly openedTokens = new Set() + private readonly identityByPty = new Map() + + constructor(publish: SshPtyOutputDataEventPublisher | undefined) { + this.coordinator = new SshPtySourceObligationCoordinator({ + onTokenClosed: (identity) => this.removeIdentity(identity), + publish: + publish ?? + ((_providerGeneration, _batch, onSettled) => + onSettled({ ok: false, error: new Error('SSH PTY source ACK publisher unavailable') })) + }) + this.remoteConsumers = new SshPtyRemoteSourceRangeConsumers(this.coordinator) + } + + get remoteHooks(): RemoteTerminalSourceRangeConsumerHooks { + return this.remoteConsumers.hooks + } + + reserve( + event: SshPtyOutputDataEvent, + projection: DesktopProjectionSpan + ): SshPtyOutputSourceReservation { + const span = this.toSourceSpan(event, projection) + const identity = this.sourceIdentity(span) + const tokenKey = ptySourceDeliveryKey(identity) + if (!this.openedTokens.has(tokenKey)) { + this.coordinator.open(identity, span.sourceStartSu) + this.openedTokens.add(tokenKey) + this.identityByPty.set(this.ptyKey(event), identity) + } + return Object.freeze({ + span, + admission: this.coordinator.reserve(identity, span, [ + 'model', + 'desktop', + ...this.remoteConsumers.requiredConsumers(event.id) + ]) + }) + } + + commit( + reservation: SshPtyOutputSourceReservation, + ptyId: string, + modelSequenceEnd: number + ): void { + this.coordinator.commit(reservation.admission) + this.remoteConsumers.trackSpan( + ptyId, + reservation.span.spanId, + reservation.admission.requiredConsumers, + modelSequenceEnd + ) + } + + rollback(reservation: SshPtyOutputSourceReservation): boolean { + return ( + this.coordinator.rollback(reservation.admission) || + this.coordinator.rollbackCommitted(reservation.admission) + ) + } + + settleModel(span: PtySourceSpan): void { + this.coordinator.settle({ + identity: span, + spanId: span.spanId, + consumer: 'model', + reason: 'model-accepted' + }) + } + + settleDesktop(span: DesktopProjectionSpan, reason: string): void { + this.coordinator.settle({ + identity: span, + spanId: span.spanId, + consumer: 'desktop', + reason + }) + } + + transferDesktop(span: DesktopProjectionSpan, reason: string): void { + const transition = { + identity: span, + spanId: span.spanId, + consumer: 'desktop' as const, + reason + } + if (this.coordinator.beginTransfer(transition, 'model')) { + this.coordinator.commitTransfer(transition) + } + } + + sealPty(event: SshPtyOutputExitEvent): void { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (identity) { + this.coordinator.seal(identity) + } + } + + markExitPublished(event: SshPtyOutputExitEvent): void { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (identity) { + this.coordinator.markExitPublished(identity) + } + } + + whenPtyTerminal(event: SshPtyOutputExitEvent): Promise { + const identity = this.identityByPty.get(this.ptyKey(event)) + return identity ? this.coordinator.whenTerminal(identity) : Promise.resolve() + } + + async requestPtyCancellationProof( + event: SshPtyOutputExitEvent, + cancel: (request: SshPtySourceCancellationRequest) => Promise + ): Promise { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (!identity) { + return null + } + const request = this.coordinator.beginExitTimeout(identity) + const proof = await cancel(request) + return Object.freeze({ identity, proof }) + } + + commitPtyCancellationProof(commit: SshPtySourceCancellationProofCommit): void { + this.coordinator.applyCancellationProof(commit.identity, commit.proof) + } + + applyCancellationProof( + event: SshPtyOutputExitEvent, + proof: SshPtySourceCancellationProof + ): boolean { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (!identity) { + return false + } + this.coordinator.applyCancellationProof(identity, proof) + return true + } + + applyRecoveryCancellationProof( + event: SshPtyOutputExitEvent, + proof: SshPtySourceCancellationProof + ): void { + const identity = this.identityByPty.get(this.ptyKey(event)) + if (identity) { + this.coordinator.applyRecoveryCancellationProof(identity, proof) + } + } + + closeGeneration(providerGeneration: number, reason: string): void { + this.remoteConsumers.closeGeneration(providerGeneration, reason) + this.coordinator.closeGeneration(providerGeneration, reason) + const prefix = `${providerGeneration}\0` + for (const key of this.openedTokens) { + if (key.startsWith(prefix)) { + this.openedTokens.delete(key) + } + } + for (const [key, identity] of this.identityByPty) { + if (identity.providerGeneration === providerGeneration) { + this.identityByPty.delete(key) + } + } + } + + acceptedCheckpoints(providerGeneration: number): readonly SshPtyAcceptedSourceCheckpoint[] { + const checkpoints: SshPtyAcceptedSourceCheckpoint[] = [] + for (const identity of this.identityByPty.values()) { + if (identity.providerGeneration !== providerGeneration) { + continue + } + checkpoints.push( + Object.freeze({ + id: identity.id, + providerGeneration: identity.providerGeneration, + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration, + ptyIncarnation: identity.ptyIncarnation, + deliveryToken: identity.deliveryToken, + acceptedSourceEndSu: this.coordinator.modelAcceptedEnd(identity) + }) + ) + } + return Object.freeze(checkpoints) + } + + acceptedCheckpoint(key: { + ptyId: string + providerGeneration: number + }): SshPtyAcceptedSourceCheckpoint | null { + return ( + this.acceptedCheckpoints(key.providerGeneration).find( + (checkpoint) => checkpoint.id === key.ptyId + ) ?? null + ) + } + + dispose(): void { + this.coordinator.dispose() + } + + getDebugSnapshot(): Readonly<{ openedTokens: number; ptyIdentities: number }> { + return Object.freeze({ + openedTokens: this.openedTokens.size, + ptyIdentities: this.identityByPty.size + }) + } + + private toSourceSpan( + event: SshPtyOutputDataEvent, + projection: DesktopProjectionSpan + ): PtySourceSpan { + return Object.freeze({ + id: projection.id, + providerGeneration: event.providerGeneration, + clientGeneration: projection.clientGeneration, + ownerGeneration: projection.ownerGeneration, + ptyIncarnation: event.ptyIncarnation, + deliveryToken: projection.deliveryToken, + spanId: projection.spanId, + sourceStartSu: projection.sourceStartSu, + sourceEndSu: projection.sourceEndSu, + displayStart: projection.displayStart, + displayEnd: projection.displayEnd, + splittable: projection.splittable, + transform: projection.transform, + data: event.data + }) + } + + private sourceIdentity(source: PtySourceSpan): PtySourceDeliveryIdentity { + return source + } + + private ptyKey(event: { + id: string + providerGeneration: number + ptyIncarnation: string + }): string { + return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}` + } + + private removeIdentity(identity: PtySourceDeliveryIdentity): void { + this.openedTokens.delete(ptySourceDeliveryKey(identity)) + for (const [key, candidate] of this.identityByPty) { + if (samePtySourceDelivery(candidate, identity)) { + this.identityByPty.delete(key) + } + } + } +} + +type SshPtyOutputDataEventPublisher = NonNullable< + SshPtyOutputIntakeDependencies['publishSourceAck'] +> diff --git a/src/main/ipc/ssh-pty-projection-terminality.ts b/src/main/ipc/ssh-pty-projection-terminality.ts new file mode 100644 index 00000000000..928ccc4301b --- /dev/null +++ b/src/main/ipc/ssh-pty-projection-terminality.ts @@ -0,0 +1,112 @@ +import type { ProjectionRecord } from './ssh-pty-legacy-projection-record' + +type ProjectionTerminalWaiter = { + providerGeneration: number + ptyIncarnation: string + resolve: () => void +} + +function matchesProjection( + record: ProjectionRecord | undefined, + providerGeneration: number, + ptyIncarnation: string +): boolean { + const identity = record?.semantics.identity + return ( + identity?.providerGeneration === providerGeneration && + identity.ptyIncarnation === ptyIncarnation + ) +} + +export function unpublishedProjectionIds( + records: ReadonlyMap, + ids: readonly string[], + providerGeneration: number, + ptyIncarnation: string +): string[] { + return ids.filter((id) => { + const record = records.get(id) + return ( + record?.state === 'committed' && matchesProjection(record, providerGeneration, ptyIncarnation) + ) + }) +} + +export function projectionHasOpen( + records: ReadonlyMap, + idsByPty: ReadonlyMap, + ptyId: string +): (providerGeneration: number, ptyIncarnation: string) => boolean { + return (providerGeneration, ptyIncarnation) => + (idsByPty.get(ptyId) ?? []).some((id) => + matchesProjection(records.get(id), providerGeneration, ptyIncarnation) + ) +} + +export function resolveProjectionTerminality( + terminality: SshPtyProjectionTerminality, + records: ReadonlyMap, + idsByPty: ReadonlyMap, + ptyId: string +): void { + terminality.resolve(ptyId, projectionHasOpen(records, idsByPty, ptyId)) +} + +export class SshPtyProjectionTerminality { + private readonly waitersByPty = new Map() + + whenTerminal( + ptyId: string, + providerGeneration: number, + ptyIncarnation: string, + hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean + ): Promise { + if (!hasOpen(providerGeneration, ptyIncarnation)) { + return Promise.resolve() + } + return new Promise((resolve) => { + const waiters = this.waitersByPty.get(ptyId) ?? [] + waiters.push({ providerGeneration, ptyIncarnation, resolve }) + this.waitersByPty.set(ptyId, waiters) + }) + } + + resolve( + ptyId: string, + hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean + ): void { + const waiters = this.waitersByPty.get(ptyId) + if (!waiters) { + return + } + const pending = waiters.filter((waiter) => + hasOpen(waiter.providerGeneration, waiter.ptyIncarnation) + ) + for (const waiter of waiters) { + if (!pending.includes(waiter)) { + waiter.resolve() + } + } + if (pending.length > 0) { + this.waitersByPty.set(ptyId, pending) + } else { + this.waitersByPty.delete(ptyId) + } + } + + closeGeneration(providerGeneration: number): void { + for (const [ptyId, waiters] of this.waitersByPty) { + const pending = waiters.filter((waiter) => waiter.providerGeneration !== providerGeneration) + for (const waiter of waiters) { + if (waiter.providerGeneration === providerGeneration) { + waiter.resolve() + } + } + if (pending.length > 0) { + this.waitersByPty.set(ptyId, pending) + } else { + this.waitersByPty.delete(ptyId) + } + } + } +} diff --git a/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts b/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts new file mode 100644 index 00000000000..9d07fb1d403 --- /dev/null +++ b/src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts @@ -0,0 +1,451 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + PtySourceDeliveryIdentity, + PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range' +import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers' +import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' + +const identity: PtySourceDeliveryIdentity = { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1' +} + +function span(spanId: string, overrides: Partial = {}): PtySourceSpan { + return { + ...identity, + spanId, + sourceStartSu: 0, + sourceEndSu: 4, + displayStart: 0, + displayEnd: 4, + data: 'data', + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }, + ...overrides + } +} + +function range( + spanId: string, + overrides: Partial = {} +): TerminalOutputSourceRange { + return { + id: 'pty-1', + spanId, + providerGeneration: 1, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4, + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }, + ...overrides + } +} + +function createCoordinator(): SshPtySourceObligationCoordinator { + return new SshPtySourceObligationCoordinator({ + publish: vi.fn(), + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) +} + +describe('SshPtyRemoteSourceRangeConsumers', () => { + it('snapshots membership and settles only the current stream generation', () => { + const ledger = createCoordinator() + const progress = vi.fn() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger, progress) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + expect(consumers.hooks.attach(stream)).toBe(true) + const sourceSpan = span('span-1') + const reservation = ledger.reserve(identity, sourceSpan, [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', sourceSpan.spanId, reservation.requiredConsumers, 4) + + consumers.hooks.settle({ ...stream, streamGeneration: 'stale' }, [range('span-1')]) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + consumers.hooks.settle(stream, [range('span-1')]) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled') + expect(progress).toHaveBeenCalledTimes(1) + }) + + it.each(['headless', 'renderer'] as const)( + 'commits remaining mappings only after a %s snapshot publication', + (source) => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + expect(replacement).not.toBeNull() + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'transferring', + to: 'remote:snapshot:consumer-1' + }) + expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 3 })).toBe(false) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('transferring') + expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 4 })).toBe(true) + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'transferred', + to: 'remote:snapshot:consumer-1' + }) + expect(consumers.requiredConsumers('pty-1')).toEqual(['remote:consumer-1']) + } + ) + + it('reserves only spans covered by the authoritative snapshot sequence', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const covered = ledger.reserve(identity, span('span-covered'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(covered) + consumers.trackSpan('pty-1', 'span-covered', covered.requiredConsumers, 4) + const trailingSpan = { + ...span('span-trailing'), + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + } + const trailing = ledger.reserve(identity, trailingSpan, [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(trailing) + consumers.trackSpan('pty-1', 'span-trailing', trailing.requiredConsumers, 8) + + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + expect(replacement).not.toBeNull() + expect(ledger.obligation('span-covered', 'remote:consumer-1')).toMatchObject({ + state: 'transferring' + }) + expect(ledger.obligation('span-trailing', 'remote:consumer-1')).toMatchObject({ + state: 'open' + }) + }) + + it('rolls a failed replacement publication back to the live stream obligation', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + expect(consumers.hooks.rollbackReplacement(replacement!, 'snapshot-write-failed')).toBe(true) + + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'open' + }) + }) + + it('rolls back every transfer when replacement reservation fails partway', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const first = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(first) + consumers.trackSpan('pty-1', 'span-1', first.requiredConsumers, 4) + const secondSpan = span('span-2', { + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + }) + const second = ledger.reserve(identity, secondSpan, [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(second) + consumers.trackSpan('pty-1', 'span-2', second.requiredConsumers, 8) + const beginTransfer = ledger.beginTransfer.bind(ledger) + vi.spyOn(ledger, 'beginTransfer').mockImplementation((transition, replacement) => { + if (transition.spanId === 'span-2') { + throw new Error('injected partial reserve failure') + } + return beginTransfer(transition, replacement) + }) + + expect(() => consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')).toThrowError( + 'injected partial reserve failure' + ) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open') + }) + + it('rejects a replacement whose exact transfer state changed before commit', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const sourceSpan = span('span-1') + const admission = ledger.reserve(identity, sourceSpan, [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(admission) + consumers.trackSpan('pty-1', 'span-1', admission.requiredConsumers, 4) + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')! + const transition = { + identity: sourceSpan, + spanId: sourceSpan.spanId, + consumer: 'remote:consumer-1' as const, + reason: 'concurrent-replacement' + } + expect(ledger.rollbackTransfer(transition)).toBe(true) + expect(ledger.beginTransfer(transition, 'remote:snapshot:consumer-1')).toBe(true) + + expect(consumers.hooks.commitReplacement(replacement, { source: 'headless', seq: 4 })).toBe( + false + ) + expect(consumers.hooks.rollbackReplacement(replacement, 'commit-rejected')).toBe(false) + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'transferring', + reason: 'concurrent-replacement' + }) + }) + + it('settles a split source span only after its complete ordered range is acknowledged', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + + consumers.hooks.settle(stream, [ + range('span-1', { + sourceEndSu: 2, + displayEnd: 2, + transform: { transformed: false, rawLengthSu: 2, scalarSafe: true } + }) + ]) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + + consumers.hooks.settle(stream, [ + range('span-1', { + sourceStartSu: 2, + sourceEndSu: 4, + displayStart: 2, + displayEnd: 4, + transform: { transformed: false, rawLengthSu: 2, scalarSafe: true } + }) + ]) + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled') + }) + + it('cancels an admitted span on detach without minting a snapshot owner', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + + consumers.hooks.cancel(stream, [], 'stream-detached') + + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'canceled', + reason: 'stream-detached' + }) + expect(consumers.requiredConsumers('pty-1')).toEqual([]) + }) + + it('rolls back a pending replacement before disconnect cancellation', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + consumers.hooks.cancel(stream, [], 'connection-closed') + + expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe( + false + ) + expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({ + state: 'canceled', + reason: 'connection-closed' + }) + }) + + it('rejects stale stream generations without changing the current obligation', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + + expect(() => + consumers.hooks.reserveReplacement( + { ...stream, streamGeneration: 'stale' }, + 4, + 'initial-snapshot' + ) + ).toThrow('stale') + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + }) + + it('rolls back replacement admission before provider-generation close', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot') + + consumers.closeGeneration(identity.providerGeneration, 'provider-replaced') + + expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open') + expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe( + false + ) + }) + + it('detaches cleanly after cancellation proof reclaims tracked spans', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + ledger.seal(identity) + ledger.beginExitTimeout(identity) + ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 }) + + expect(() => consumers.hooks.settle(stream, [range('span-1')])).not.toThrow() + expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow() + expect(consumers.requiredConsumers('pty-1')).toEqual([]) + }) + + it('rejects and rolls back a replacement after proof reclaims its covered spans', () => { + const ledger = createCoordinator() + const consumers = new SshPtyRemoteSourceRangeConsumers(ledger) + const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' } + ledger.open(identity) + consumers.hooks.attach(stream) + const reservation = ledger.reserve(identity, span('span-1'), [ + 'model', + ...consumers.requiredConsumers('pty-1') + ]) + ledger.commit(reservation) + consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4) + const recoveredIdentity = { + ...identity, + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-2' + } + ledger.open(recoveredIdentity, 4) + const recovered = ledger.reserve( + recoveredIdentity, + span('span-2', { + ...recoveredIdentity, + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + }), + ['model', ...consumers.requiredConsumers('pty-1')] + ) + ledger.commit(recovered) + consumers.trackSpan('pty-1', 'span-2', recovered.requiredConsumers, 8) + const replacement = consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot') + ledger.seal(identity) + ledger.beginExitTimeout(identity) + ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 }) + + const queuedOutput = ['covered-output', 'surviving-output'] + const committed = consumers.hooks.commitReplacement(replacement!, { + source: 'headless', + seq: 8 + }) + if (committed) { + queuedOutput.splice(0) + } + + expect(committed).toBe(false) + expect(queuedOutput).toEqual(['covered-output', 'surviving-output']) + expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(true) + expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open') + expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(false) + expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow() + expect(consumers.requiredConsumers('pty-1')).toEqual([]) + }) +}) diff --git a/src/main/ipc/ssh-pty-remote-source-range-consumers.ts b/src/main/ipc/ssh-pty-remote-source-range-consumers.ts new file mode 100644 index 00000000000..5ddb1e8d44b --- /dev/null +++ b/src/main/ipc/ssh-pty-remote-source-range-consumers.ts @@ -0,0 +1,285 @@ +import { + sameTerminalOutputSourceIdentity, + type TerminalOutputSourceRange +} from '../../shared/terminal-output-source-range' +import type { + RemoteTerminalSourceRangeConsumerHooks, + RemoteTerminalSourceRangeStreamIdentity +} from '../runtime/remote-terminal-source-range-consumer' +import type { PtySourceSpan } from '../../shared/pty-source-credit-contract' +import type { SshPtySourceConsumerId } from './ssh-pty-source-obligation-contract' +import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' +import { SshPtyRemoteSourceRangeReplacements } from './ssh-pty-remote-source-range-replacement' + +function remoteConsumerId( + identity: RemoteTerminalSourceRangeStreamIdentity +): SshPtySourceConsumerId { + return `remote:${identity.consumerId}` +} + +function uniqueSpanIds(ranges: readonly TerminalOutputSourceRange[]): string[] { + return Array.from(new Set(ranges.map((range) => range.spanId))) +} + +type RemoteConsumerState = { + streamGeneration: string + spans: Map< + string, + Readonly<{ + identity: PtySourceSpan + modelSequenceEnd: number + }> + > + ackedEndBySpan: Map +} + +export class SshPtyRemoteSourceRangeConsumers { + private readonly consumersByPty = new Map>() + private readonly replacements: SshPtyRemoteSourceRangeReplacements + + constructor( + private readonly coordinator: SshPtySourceObligationCoordinator, + private readonly onProgress: (range: TerminalOutputSourceRange) => void = () => {} + ) { + this.replacements = new SshPtyRemoteSourceRangeReplacements(coordinator) + } + + readonly hooks: RemoteTerminalSourceRangeConsumerHooks = { + attach: (identity) => this.attach(identity), + settle: (identity, ranges) => this.settle(identity, ranges), + reserveReplacement: (identity, requiredSeq, reason) => + this.reserveReplacement(identity, requiredSeq, reason), + commitReplacement: (reservation, publication) => + this.commitReplacement(reservation, publication), + rollbackReplacement: (reservation, reason) => this.rollbackReplacement(reservation, reason), + cancel: (identity, ranges, reason) => this.cancel(identity, ranges, reason) + } + + requiredConsumers(ptyId: string): readonly SshPtySourceConsumerId[] { + return Object.freeze( + Array.from(this.consumersByPty.get(ptyId)?.keys() ?? []).map( + (consumerId) => `remote:${consumerId}` as const + ) + ) + } + + trackSpan( + ptyId: string, + spanId: string, + requiredConsumers: readonly SshPtySourceConsumerId[], + modelSequenceEnd: number + ): void { + if (!Number.isSafeInteger(modelSequenceEnd) || modelSequenceEnd < 0) { + throw new Error('ssh_remote_source_range_model_sequence_invalid') + } + for (const [consumerId, state] of this.consumersByPty.get(ptyId) ?? []) { + if (requiredConsumers.includes(`remote:${consumerId}`)) { + state.spans.set( + spanId, + Object.freeze({ + identity: this.coordinator.spanIdentity(spanId), + modelSequenceEnd + }) + ) + } + } + } + + closeGeneration(providerGeneration: number, reason: string): void { + this.replacements.closeGeneration(providerGeneration, reason) + for (const consumers of this.consumersByPty.values()) { + for (const state of consumers.values()) { + for (const [spanId, tracked] of state.spans) { + if (tracked.identity.providerGeneration === providerGeneration) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + } + } + } + } + } + + private attach(identity: RemoteTerminalSourceRangeStreamIdentity): boolean { + const consumers = + this.consumersByPty.get(identity.ptyId) ?? new Map() + const current = consumers.get(identity.consumerId) + if (current && current.streamGeneration !== identity.streamGeneration) { + return false + } + consumers.set( + identity.consumerId, + current ?? { + streamGeneration: identity.streamGeneration, + spans: new Map(), + ackedEndBySpan: new Map() + } + ) + this.consumersByPty.set(identity.ptyId, consumers) + return true + } + + private settle( + identity: RemoteTerminalSourceRangeStreamIdentity, + ranges: readonly TerminalOutputSourceRange[] + ): void { + if (!this.isCurrent(identity)) { + return + } + const state = this.requireState(identity) + const consumer = remoteConsumerId(identity) + const nextEnds = new Map(state.ackedEndBySpan) + const completed = new Set() + for (const range of ranges) { + const tracked = state.spans.get(range.spanId) + if (!tracked) { + continue + } + const source = tracked.identity + if (!this.coordinator.hasRetainedSpan(range.spanId)) { + state.spans.delete(range.spanId) + nextEnds.delete(range.spanId) + continue + } + const currentEnd = nextEnds.get(range.spanId) ?? source.sourceStartSu + if ( + !sameTerminalOutputSourceIdentity(source, range) || + range.sourceStartSu !== currentEnd || + range.sourceEndSu > source.sourceEndSu + ) { + throw new Error('ssh_remote_source_range_settlement_invalid') + } + nextEnds.set(range.spanId, range.sourceEndSu) + if (range.sourceEndSu === source.sourceEndSu) { + completed.add(range.spanId) + } + } + state.ackedEndBySpan = nextEnds + for (const spanId of completed) { + const tracked = state.spans.get(spanId) + if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + continue + } + const source = tracked.identity + this.coordinator.settle({ + identity: source, + spanId, + consumer, + reason: 'remote-frame-ack' + }) + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + } + for (const range of ranges) { + this.onProgress(range) + } + } + + private reserveReplacement( + identity: RemoteTerminalSourceRangeStreamIdentity, + requiredSeq: number, + reason: string + ): ReturnType { + if (!this.isCurrent(identity)) { + throw new Error('ssh_remote_source_range_stale_generation') + } + const state = this.requireState(identity) + for (const spanId of state.spans.keys()) { + if (!this.coordinator.hasRetainedSpan(spanId)) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + } + } + const spanIds = Array.from(state.spans) + .filter(([, tracked]) => tracked.modelSequenceEnd <= requiredSeq) + .map(([spanId]) => spanId) + return this.replacements.reserve(identity, spanIds, requiredSeq, reason) + } + + private commitReplacement( + reservation: Parameters[0], + publication: Parameters[1] + ): boolean { + return this.replacements.commit( + reservation, + publication, + this.isCurrent(reservation.identity), + (spanIds) => { + const state = this.requireState(reservation.identity) + for (const spanId of spanIds) { + state.spans.delete(spanId) + state.ackedEndBySpan.delete(spanId) + } + } + ) + } + + private rollbackReplacement( + reservation: Parameters[0], + reason: string + ): boolean { + return this.replacements.rollback(reservation, reason) + } + + private cancel( + identity: RemoteTerminalSourceRangeStreamIdentity, + ranges: readonly TerminalOutputSourceRange[], + reason: string + ): void { + if (!this.isCurrent(identity)) { + return + } + this.replacements.rollbackIdentity(identity, `${reason}-replacement-aborted`) + const consumer = remoteConsumerId(identity) + const state = this.requireState(identity) + const spanIds = new Set([ + ...state.spans.keys(), + ...uniqueSpanIds(ranges).filter((spanId) => state.spans.has(spanId)) + ]) + for (const spanId of spanIds) { + const tracked = state.spans.get(spanId) + if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) { + continue + } + const source = tracked.identity + const transition = { identity: source, spanId, consumer, reason } + if (this.coordinator.beginTransfer(transition, consumer)) { + this.coordinator.cancelTransfer(transition) + } + } + this.detachIdentity(identity) + for (const range of ranges) { + this.onProgress(range) + } + } + + private isCurrent(identity: RemoteTerminalSourceRangeStreamIdentity): boolean { + return ( + this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)?.streamGeneration === + identity.streamGeneration + ) + } + + private requireState(identity: RemoteTerminalSourceRangeStreamIdentity): RemoteConsumerState { + const state = this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId) + if (!state || state.streamGeneration !== identity.streamGeneration) { + throw new Error('ssh_remote_source_range_stale_generation') + } + return state + } + + private detachIdentity(identity: RemoteTerminalSourceRangeStreamIdentity): void { + const consumers = this.consumersByPty.get(identity.ptyId) + if ( + !consumers || + consumers.get(identity.consumerId)?.streamGeneration !== identity.streamGeneration + ) { + return + } + consumers.delete(identity.consumerId) + if (consumers.size === 0) { + this.consumersByPty.delete(identity.ptyId) + } + } +} diff --git a/src/main/ipc/ssh-pty-remote-source-range-replacement.ts b/src/main/ipc/ssh-pty-remote-source-range-replacement.ts new file mode 100644 index 00000000000..5cb60311c25 --- /dev/null +++ b/src/main/ipc/ssh-pty-remote-source-range-replacement.ts @@ -0,0 +1,194 @@ +import type { + RemoteTerminalSourceRangeReplacementPublication, + RemoteTerminalSourceRangeReplacementReservation, + RemoteTerminalSourceRangeStreamIdentity +} from '../runtime/remote-terminal-source-range-consumer' +import type { PtySourceSpan } from '../../shared/pty-source-credit-contract' +import type { + SshPtySourceConsumerId, + SshPtySourceObligationState +} from './ssh-pty-source-obligation-contract' +import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' + +type ReplacementSpanRecord = Readonly<{ + source: PtySourceSpan + transferState: SshPtySourceObligationState +}> + +type ReplacementReservationRecord = { + reservation: RemoteTerminalSourceRangeReplacementReservation + spans: readonly ReplacementSpanRecord[] + consumer: SshPtySourceConsumerId + replacement: SshPtySourceConsumerId + reason: string +} + +function remoteConsumerId( + identity: RemoteTerminalSourceRangeStreamIdentity +): SshPtySourceConsumerId { + return `remote:${identity.consumerId}` +} + +export class SshPtyRemoteSourceRangeReplacements { + private readonly reservations = new Map() + private nextReservationId = 1 + + constructor(private readonly coordinator: SshPtySourceObligationCoordinator) {} + + reserve( + identity: RemoteTerminalSourceRangeStreamIdentity, + spanIds: readonly string[], + requiredSeq: number, + reason: string + ): RemoteTerminalSourceRangeReplacementReservation | null { + if (spanIds.length === 0) { + return null + } + if (!Number.isSafeInteger(requiredSeq) || requiredSeq < 0) { + throw new Error('ssh_remote_source_range_replacement_sequence_invalid') + } + const consumer = remoteConsumerId(identity) + const replacement = `remote:snapshot:${identity.consumerId}` as const + const spans = spanIds.map((spanId) => this.coordinator.spanIdentity(spanId)) + for (const { spanId } of spans) { + if (this.coordinator.obligation(spanId, consumer).state !== 'open') { + throw new Error('ssh_remote_source_range_transfer_invalid') + } + } + const transferred: ReplacementSpanRecord[] = [] + try { + for (const source of spans) { + const { spanId } = source + const transition = { identity: source, spanId, consumer, reason } + if (!this.coordinator.beginTransfer(transition, replacement)) { + throw new Error('ssh_remote_source_range_transfer_invalid') + } + const transferState = this.coordinator.obligation(spanId, consumer) + if (transferState.state !== 'transferring' || transferState.to !== replacement) { + this.coordinator.rollbackTransfer(transition) + throw new Error('ssh_remote_source_range_transfer_invalid') + } + transferred.push(Object.freeze({ source, transferState })) + } + } catch (error) { + for (const span of transferred.toReversed()) { + this.rollbackExactSpan(span, consumer, reason) + } + throw error + } + const reservation = Object.freeze({ + reservationId: `remote-source-replacement:${this.nextReservationId++}`, + identity: Object.freeze({ ...identity }), + requiredSeq + }) + this.reservations.set(reservation.reservationId, { + reservation, + spans: Object.freeze(transferred), + consumer, + replacement, + reason + }) + return reservation + } + + commit( + reservation: RemoteTerminalSourceRangeReplacementReservation, + publication: RemoteTerminalSourceRangeReplacementPublication, + isCurrent: boolean, + onCommitted: (spanIds: readonly string[]) => void + ): boolean { + const record = this.reservations.get(reservation.reservationId) + if ( + !record || + record.reservation !== reservation || + !isCurrent || + !Number.isSafeInteger(publication.seq) || + publication.seq < reservation.requiredSeq || + (publication.source !== 'headless' && publication.source !== 'renderer') + ) { + return false + } + if ( + record.spans.some(({ source, transferState }) => { + const { spanId } = source + if (!this.coordinator.hasRetainedSpan(spanId)) { + return true + } + return this.coordinator.obligation(spanId, record.consumer) !== transferState + }) + ) { + return false + } + for (const { source } of record.spans) { + const { spanId } = source + if (!this.coordinator.hasRetainedSpan(spanId)) { + continue + } + if ( + !this.coordinator.commitTransfer({ identity: source, spanId, consumer: record.consumer }) + ) { + throw new Error('ssh_remote_source_range_replacement_commit_invalid') + } + } + this.reservations.delete(reservation.reservationId) + onCommitted(record.spans.map(({ source }) => source.spanId)) + return true + } + + rollback(reservation: RemoteTerminalSourceRangeReplacementReservation, reason: string): boolean { + const record = this.reservations.get(reservation.reservationId) + if (!record || record.reservation !== reservation) { + return false + } + this.reservations.delete(reservation.reservationId) + let rolledBack = true + for (const span of record.spans) { + const { spanId } = span.source + if (!this.coordinator.hasRetainedSpan(spanId)) { + continue + } + rolledBack = this.rollbackExactSpan(span, record.consumer, reason) && rolledBack + } + return rolledBack + } + + rollbackIdentity(identity: RemoteTerminalSourceRangeStreamIdentity, reason: string): void { + for (const record of Array.from(this.reservations.values())) { + if ( + record.reservation.identity.ptyId === identity.ptyId && + record.reservation.identity.consumerId === identity.consumerId && + record.reservation.identity.streamGeneration === identity.streamGeneration + ) { + this.rollback(record.reservation, reason) + } + } + } + + closeGeneration(providerGeneration: number, reason: string): void { + for (const record of Array.from(this.reservations.values())) { + if (record.spans.some(({ source }) => source.providerGeneration === providerGeneration)) { + this.rollback(record.reservation, `${reason}-replacement-aborted`) + } + } + } + + private rollbackExactSpan( + span: ReplacementSpanRecord, + consumer: SshPtySourceConsumerId, + reason: string + ): boolean { + const { source, transferState } = span + if ( + !this.coordinator.hasRetainedSpan(source.spanId) || + this.coordinator.obligation(source.spanId, consumer) !== transferState + ) { + return false + } + return this.coordinator.rollbackTransfer({ + identity: source, + spanId: source.spanId, + consumer, + reason + }) + } +} diff --git a/src/main/ipc/ssh-pty-source-ack-coalescer.test.ts b/src/main/ipc/ssh-pty-source-ack-coalescer.test.ts new file mode 100644 index 00000000000..9e0267c5f5d --- /dev/null +++ b/src/main/ipc/ssh-pty-source-ack-coalescer.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it, vi } from 'vitest' +import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract' +import { SshPtySourceAckCoalescer } from './ssh-pty-source-ack-coalescer' + +function publication(token: number, endSu: number, settled = vi.fn(), providerGeneration = 1) { + const identity = { + id: `pty-${token}`, + providerGeneration, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: `incarnation-${token}`, + deliveryToken: `token-${token}` + } + return { + identity, + ack: { + id: identity.id, + clientGeneration: 1, + ownerGeneration: 1, + deliveryToken: identity.deliveryToken, + creditedEndSu: endSu + }, + onSettled: settled + } +} + +describe('SshPtySourceAckCoalescer', () => { + it('coalesces cumulative values and advances them only from the write callback', () => { + const writes: { + batch: PtySourceCreditAckBatch + settle: (result: { ok: true } | { ok: false; error: Error }) => void + }[] = [] + const firstSettled = vi.fn() + const latestSettled = vi.fn() + const coalescer = new SshPtySourceAckCoalescer({ + publish: (_providerGeneration, batch, settle) => writes.push({ batch, settle }), + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coalescer.enqueue(publication(1, 10, firstSettled)) + coalescer.enqueue(publication(1, 20, latestSettled)) + coalescer.flush() + + expect(writes[0].batch.acknowledgements).toEqual([ + expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 20 }) + ]) + expect(latestSettled).not.toHaveBeenCalled() + writes[0].settle({ ok: true }) + expect(latestSettled).toHaveBeenCalledWith({ ok: true }) + expect(firstSettled).toHaveBeenCalledWith({ ok: true }) + }) + + it('limits one batch to 64 tokens and gives the remainder another turn', () => { + const batches: PtySourceCreditAckBatch[] = [] + const scheduled: (() => void)[] = [] + const coalescer = new SshPtySourceAckCoalescer({ + publish: (_providerGeneration, batch, settle) => { + batches.push(batch) + settle({ ok: true }) + }, + schedule: (callback) => { + scheduled.push(callback) + return scheduled.length as unknown as ReturnType + }, + cancelSchedule: vi.fn() + }) + for (let token = 0; token < 70; token++) { + coalescer.enqueue(publication(token, 1)) + } + + coalescer.flush() + expect(batches[0].acknowledgements).toHaveLength(64) + scheduled.at(-1)!() + expect(batches[1].acknowledgements).toHaveLength(6) + }) + + it('never mixes provider generations in one transport batch', () => { + const generations: number[] = [] + const coalescer = new SshPtySourceAckCoalescer({ + publish: (providerGeneration, _batch, settle) => { + generations.push(providerGeneration) + settle({ ok: true }) + }, + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coalescer.enqueue(publication(1, 1, vi.fn(), 1)) + coalescer.enqueue(publication(2, 1, vi.fn(), 2)) + + coalescer.flush() + coalescer.flush() + + expect(generations).toEqual([1, 2]) + }) + + it('settles every entry as failed on a synchronous send error', () => { + const settled = vi.fn() + const coalescer = new SshPtySourceAckCoalescer({ + publish: () => { + throw new Error('send failed') + }, + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coalescer.enqueue(publication(1, 10, settled)) + coalescer.flush() + + expect(settled).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ message: 'send failed' }) + }) + }) + + it('promotes a pending interval flush to immediate at the source threshold', () => { + const delays: number[] = [] + const cancelSchedule = vi.fn() + const coalescer = new SshPtySourceAckCoalescer({ + publish: vi.fn(), + schedule: (_callback, delayMs) => { + delays.push(delayMs) + return delays.length as unknown as ReturnType + }, + cancelSchedule + }) + + coalescer.enqueue(publication(1, 1)) + coalescer.enqueue(publication(1, 64 * 1024)) + + expect(delays).toEqual([8, 0]) + expect(cancelSchedule).toHaveBeenCalledOnce() + }) + + it('fails queued callbacks exactly once during cleanup', () => { + const settled = vi.fn() + const coalescer = new SshPtySourceAckCoalescer({ + publish: vi.fn(), + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coalescer.enqueue(publication(1, 10, settled)) + + coalescer.dispose() + coalescer.dispose() + expect(settled).toHaveBeenCalledOnce() + }) + + it('owns an in-flight callback until dispose and ignores its late transport callback', () => { + let transportSettle!: (result: { ok: true } | { ok: false; error: Error }) => void + const settled = vi.fn() + const coalescer = new SshPtySourceAckCoalescer({ + publish: (_providerGeneration, _batch, settle) => { + transportSettle = settle + }, + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coalescer.enqueue(publication(1, 10, settled)) + coalescer.flush() + + coalescer.dispose('generation closed') + transportSettle({ ok: true }) + expect(settled).toHaveBeenCalledOnce() + expect(settled).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ message: 'generation closed' }) + }) + }) +}) diff --git a/src/main/ipc/ssh-pty-source-ack-coalescer.ts b/src/main/ipc/ssh-pty-source-ack-coalescer.ts new file mode 100644 index 00000000000..2bb55130688 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-ack-coalescer.ts @@ -0,0 +1,230 @@ +import { + MAX_PTY_ACK_ENTRIES, + type PtySourceCreditAckBatch +} from '../../shared/pty-source-credit-contract' +import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract' + +export const SSH_PTY_ACK_FLUSH_MS = 8 +export const SSH_PTY_ACK_EAGER_ADVANCE_SU = 64 * 1024 +const ACK_PUBLICATION_WATERMARK_LIMIT = 1024 + +type AckSettlement = { ok: true } | { ok: false; error: Error } + +type CoalescedEntry = { + publication: SshPtySourceAckPublication + members: SshPtySourceAckPublication[] +} + +type InFlightBatch = { + entries: CoalescedEntry[] + settled: boolean +} + +export type SshPtySourceAckCoalescerOptions = { + publish: ( + providerGeneration: number, + batch: PtySourceCreditAckBatch, + onSettled: (result: AckSettlement) => void + ) => void + schedule?: (callback: () => void, delayMs: number) => ReturnType + cancelSchedule?: (timer: ReturnType) => void + onTokenClosed?: (identity: SshPtySourceAckPublication['identity']) => void +} + +function ackKey(publication: SshPtySourceAckPublication): string { + const { identity } = publication + return `${identity.providerGeneration}\0${identity.clientGeneration}\0${identity.ownerGeneration}\0${identity.id}\0${identity.ptyIncarnation}\0${identity.deliveryToken}` +} + +export class SshPtySourceAckCoalescer { + private readonly pending = new Map() + private readonly lastPublishedEndByToken = new Map() + private readonly schedule: NonNullable + private readonly cancelSchedule: NonNullable + private timer: ReturnType | null = null + private timerDelayMs: number | null = null + private inFlight: InFlightBatch | null = null + private disposed = false + + constructor(private readonly options: SshPtySourceAckCoalescerOptions) { + this.schedule = options.schedule ?? ((callback, delayMs) => setTimeout(callback, delayMs)) + this.cancelSchedule = options.cancelSchedule ?? clearTimeout + } + + enqueue(publication: SshPtySourceAckPublication): void { + if (this.disposed) { + publication.onSettled({ ok: false, error: new Error('SSH PTY ACK coalescer disposed') }) + return + } + const key = ackKey(publication) + const current = this.pending.get(key) + if (!current) { + this.pending.set(key, { publication, members: [publication] }) + } else { + current.members.push(publication) + if (publication.ack.creditedEndSu > current.publication.ack.creditedEndSu) { + current.publication = publication + } + } + const lastPublished = this.lastPublishedEndByToken.get(key) ?? 0 + const eager = publication.ack.creditedEndSu - lastPublished >= SSH_PTY_ACK_EAGER_ADVANCE_SU + this.requestFlush(eager ? 0 : SSH_PTY_ACK_FLUSH_MS) + } + + flush(): void { + if (this.disposed || this.inFlight || this.pending.size === 0) { + return + } + if (this.timer) { + this.cancelSchedule(this.timer) + this.timer = null + this.timerDelayMs = null + } + const providerGeneration = this.pending.values().next().value!.publication + .identity.providerGeneration + const selected = Array.from(this.pending.entries()) + .filter(([, entry]) => entry.publication.identity.providerGeneration === providerGeneration) + .slice(0, MAX_PTY_ACK_ENTRIES) + for (const [key] of selected) { + this.pending.delete(key) + } + const batch: InFlightBatch = { + entries: selected.map(([, entry]) => entry), + settled: false + } + this.inFlight = batch + const settle = (result: AckSettlement): void => this.settleBatch(batch, result) + try { + this.options.publish( + providerGeneration, + Object.freeze({ + acknowledgements: Object.freeze(batch.entries.map((entry) => entry.publication.ack)) + }), + settle + ) + } catch (error) { + settle({ + ok: false, + error: error instanceof Error ? error : new Error(String(error)) + }) + } + } + + dispose(reason = 'SSH PTY ACK coalescer disposed'): void { + if (this.disposed) { + return + } + this.disposed = true + if (this.timer) { + this.cancelSchedule(this.timer) + this.timer = null + this.timerDelayMs = null + } + const result = { ok: false as const, error: new Error(reason) } + for (const entry of this.pending.values()) { + this.settleMembers(entry, result) + } + this.pending.clear() + if (this.inFlight && !this.inFlight.settled) { + const batch = this.inFlight + batch.settled = true + this.inFlight = null + for (const entry of batch.entries) { + this.settleMembers(entry, result) + } + } + } + + cancelGeneration(providerGeneration: number, reason: string): void { + const result = { ok: false as const, error: new Error(reason) } + for (const [key, entry] of this.pending) { + if (entry.publication.identity.providerGeneration === providerGeneration) { + this.pending.delete(key) + this.settleMembers(entry, result) + } + } + const batch = this.inFlight + if (batch && !batch.settled) { + const retained: CoalescedEntry[] = [] + for (const entry of batch.entries) { + if (entry.publication.identity.providerGeneration === providerGeneration) { + this.settleMembers(entry, result) + } else { + retained.push(entry) + } + } + batch.entries = retained + if (retained.length === 0) { + batch.settled = true + this.inFlight = null + } + } + const prefix = `${providerGeneration}\0` + for (const key of this.lastPublishedEndByToken.keys()) { + if (key.startsWith(prefix)) { + this.lastPublishedEndByToken.delete(key) + } + } + if (!this.inFlight && this.pending.size === 0 && this.timer) { + this.cancelSchedule(this.timer) + this.timer = null + this.timerDelayMs = null + } else if (!this.inFlight && this.pending.size > 0) { + this.requestFlush(0) + } + } + + get pendingCount(): number { + return this.pending.size + } + + private settleBatch(batch: InFlightBatch, result: AckSettlement): void { + if (batch.settled) { + return + } + batch.settled = true + if (this.inFlight === batch) { + this.inFlight = null + } + for (const entry of batch.entries) { + if (result.ok) { + const key = ackKey(entry.publication) + this.lastPublishedEndByToken.delete(key) + this.lastPublishedEndByToken.set(key, entry.publication.ack.creditedEndSu) + while (this.lastPublishedEndByToken.size > ACK_PUBLICATION_WATERMARK_LIMIT) { + this.lastPublishedEndByToken.delete(this.lastPublishedEndByToken.keys().next().value!) + } + } + this.settleMembers(entry, result) + } + if (this.pending.size > 0) { + this.requestFlush(0) + } + } + + private settleMembers(entry: CoalescedEntry, result: AckSettlement): void { + for (const publication of entry.members.splice(0)) { + publication.onSettled(result) + } + } + + private requestFlush(delayMs: number): void { + if (this.inFlight || this.disposed) { + return + } + if (this.timer) { + if (this.timerDelayMs !== null && this.timerDelayMs <= delayMs) { + return + } + this.cancelSchedule(this.timer) + this.timer = null + } + this.timerDelayMs = delayMs + this.timer = this.schedule(() => { + this.timer = null + this.timerDelayMs = null + this.flush() + }, delayMs) + this.timer.unref?.() + } +} diff --git a/src/main/ipc/ssh-pty-source-ack-publication.ts b/src/main/ipc/ssh-pty-source-ack-publication.ts new file mode 100644 index 00000000000..db1d9581255 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-ack-publication.ts @@ -0,0 +1,33 @@ +import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract' +import { reclaimPublishedSourcePrefix, type TokenRecord } from './ssh-pty-source-obligation-state' + +export function createSshPtySourceAckPublication( + token: TokenRecord, + endSu: number, + spanOwners: Map, + onPublished: () => void +): SshPtySourceAckPublication { + let settled = false + return Object.freeze({ + identity: token.identity, + ack: Object.freeze({ + id: token.identity.id, + clientGeneration: token.identity.clientGeneration, + ownerGeneration: token.identity.ownerGeneration, + deliveryToken: token.identity.deliveryToken, + creditedEndSu: endSu + }), + onSettled: (result) => { + if (settled) { + return + } + settled = true + if (!result.ok || token.state === 'closed' || endSu > token.ackQueuedEndSu) { + return + } + token.ackPublishedEndSu = Math.max(token.ackPublishedEndSu, endSu) + reclaimPublishedSourcePrefix(token, spanOwners) + onPublished() + } + }) +} diff --git a/src/main/ipc/ssh-pty-source-ack-session-contract.test.ts b/src/main/ipc/ssh-pty-source-ack-session-contract.test.ts new file mode 100644 index 00000000000..9ee6f90c5a3 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-ack-session-contract.test.ts @@ -0,0 +1,86 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract' +import { SshPtyOutputIntake } from './ssh-pty-output-intake' +import { + installSshPtySourceAckPublisher, + publishSshPtySourceAck +} from './ssh-pty-output-intake-registry' + +afterEach(() => vi.useRealTimers()) + +describe('SSH PTY intake to relay ACK contract', () => { + it('publishes one cumulative relay-ID ACK only after model and desktop settlement', async () => { + vi.useFakeTimers() + const batches: PtySourceCreditAckBatch[] = [] + const cleanup = installSshPtySourceAckPublisher(7, (batch, onSettled) => { + batches.push(batch) + onSettled({ ok: true }) + }) + let sequence = 0 + const intake = new SshPtyOutputIntake({ + getModelSequence: () => sequence, + acceptModel: (event) => { + sequence += event.rawLength + return { sequence, completion: Promise.resolve() } + }, + project: () => {}, + prepareExit: () => {}, + finalizeExit: () => {}, + publishSourceAck: publishSshPtySourceAck + }) + + try { + const receipt = await intake.acceptData({ + id: 'ssh:target@@relay-pty-1', + data: 'data', + providerGeneration: 7, + ptyIncarnation: 'incarnation-1', + rawLength: 4, + transformed: false, + source: { + relayPtyId: 'relay-pty-1', + spanId: 'token-1:0:4', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + await vi.advanceTimersByTimeAsync(8) + expect(batches).toHaveLength(0) + + const projectionId = receipt.projection.identity.projectionSemanticsId + intake.publishProjectionPrefix([projectionId], 4, 4) + intake.settleProjectionPrefix('ssh:target@@relay-pty-1', 4) + await vi.advanceTimersByTimeAsync(8) + + expect(batches).toEqual([ + { + acknowledgements: [ + { + id: 'relay-pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + creditedEndSu: 4 + } + ] + } + ]) + await intake.acceptExit({ + id: 'ssh:target@@relay-pty-1', + code: 0, + providerGeneration: 7, + ptyIncarnation: 'incarnation-1' + }) + expect(intake.getDebugSnapshot().source).toEqual({ + openedTokens: 0, + ptyIdentities: 0 + }) + } finally { + intake.dispose() + cleanup() + } + }) +}) diff --git a/src/main/ipc/ssh-pty-source-obligation-contract.ts b/src/main/ipc/ssh-pty-source-obligation-contract.ts new file mode 100644 index 00000000000..28429bfd5c4 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-contract.ts @@ -0,0 +1,38 @@ +import type { + PtySourceCreditAck, + PtySourceDeliveryIdentity, + PtySourceSpan +} from '../../shared/pty-source-credit-contract' + +export type SshPtySourceConsumerId = 'model' | 'desktop' | `remote:${string}` + +export type SshPtySourceObligationState = + | Readonly<{ state: 'open' }> + | Readonly<{ state: 'transferring'; to: SshPtySourceConsumerId; reason: string }> + | Readonly<{ state: 'settled'; reason: string }> + | Readonly<{ state: 'transferred'; to: SshPtySourceConsumerId; reason: string }> + | Readonly<{ state: 'canceled'; reason: string }> + +export type SshPtySourceAdmissionReservation = Readonly<{ + reservationId: string + span: PtySourceSpan + requiredConsumers: readonly SshPtySourceConsumerId[] +}> + +export type SshPtySourceTokenSnapshot = PtySourceDeliveryIdentity & + Readonly<{ + state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed' + receivedEndSu: number + obligationsTerminalEndSu: number + ackQueuedEndSu: number + ackPublishedEndSu: number + openSpans: number + exitPublished: boolean + generationClosed: boolean + }> + +export type SshPtySourceAckPublication = Readonly<{ + identity: PtySourceDeliveryIdentity + ack: PtySourceCreditAck + onSettled: (result: { ok: true } | { ok: false; error: Error }) => void +}> diff --git a/src/main/ipc/ssh-pty-source-obligation-coordinator.test.ts b/src/main/ipc/ssh-pty-source-obligation-coordinator.test.ts new file mode 100644 index 00000000000..7c767dbf46a --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-coordinator.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + PtySourceCreditAckBatch, + PtySourceDeliveryIdentity, + PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator' + +const identity: PtySourceDeliveryIdentity = { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1' +} + +const span: PtySourceSpan = { + ...identity, + spanId: 'span-1', + sourceStartSu: 0, + sourceEndSu: 4, + displayStart: 0, + displayEnd: 4, + data: 'data', + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } +} + +describe('SshPtySourceObligationCoordinator', () => { + it('is the single boundary from exact consumer settlement to upstream ACK publication', () => { + let written: + | { + batch: PtySourceCreditAckBatch + settle: (result: { ok: true } | { ok: false; error: Error }) => void + } + | undefined + const coordinator = new SshPtySourceObligationCoordinator({ + publish: (_providerGeneration, batch, settle) => { + written = { batch, settle } + }, + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coordinator.open(identity) + const reservation = coordinator.reserve(identity, span, ['model', 'desktop']) + coordinator.commit(reservation) + coordinator.settle({ + identity, + spanId: span.spanId, + consumer: 'model', + reason: 'emulator-receipt' + }) + coordinator.settle({ + identity, + spanId: span.spanId, + consumer: 'desktop', + reason: 'renderer-parse' + }) + coordinator.flushAcknowledgements() + + expect(written?.batch.acknowledgements).toEqual([ + expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 4 }) + ]) + expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(0) + written?.settle({ ok: true }) + expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(4) + }) + + it('rejects an adapter transition carrying a stale full delivery identity', () => { + const coordinator = new SshPtySourceObligationCoordinator({ + publish: vi.fn(), + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coordinator.open(identity) + coordinator.commit(coordinator.reserve(identity, span, ['model'])) + + expect(() => + coordinator.settle({ + identity: { ...identity, ownerGeneration: 99 }, + spanId: span.spanId, + consumer: 'model', + reason: 'stale' + }) + ).toThrow('stale') + }) + + it('keeps later generations publishable after closing one generation', () => { + const publish = vi.fn() + const coordinator = new SshPtySourceObligationCoordinator({ + publish, + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coordinator.open(identity) + coordinator.closeGeneration(1, 'replaced') + const nextIdentity = { ...identity, providerGeneration: 2, deliveryToken: 'token-2' } + const nextSpan = { ...span, ...nextIdentity, spanId: 'span-2' } + coordinator.open(nextIdentity) + coordinator.commit(coordinator.reserve(nextIdentity, nextSpan, ['model'])) + coordinator.settle({ + identity: nextIdentity, + spanId: nextSpan.spanId, + consumer: 'model', + reason: 'accepted' + }) + coordinator.flushAcknowledgements() + + expect(publish).toHaveBeenCalledWith( + 2, + { + acknowledgements: [expect.objectContaining({ deliveryToken: 'token-2', creditedEndSu: 4 })] + }, + expect.any(Function) + ) + }) + + it('generation-closes every retained token on coordinator disposal', () => { + const coordinator = new SshPtySourceObligationCoordinator({ + publish: vi.fn(), + schedule: vi.fn(() => 1 as unknown as ReturnType), + cancelSchedule: vi.fn() + }) + coordinator.open(identity) + coordinator.commit(coordinator.reserve(identity, span, ['model'])) + + coordinator.dispose('provider disposed') + + expect(coordinator.snapshot(identity)).toMatchObject({ + state: 'closed', + generationClosed: true, + openSpans: 0 + }) + expect(() => coordinator.open({ ...identity, deliveryToken: 'late-token' })).toThrow('disposed') + }) +}) diff --git a/src/main/ipc/ssh-pty-source-obligation-coordinator.ts b/src/main/ipc/ssh-pty-source-obligation-coordinator.ts new file mode 100644 index 00000000000..557e1799a50 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-coordinator.ts @@ -0,0 +1,267 @@ +import { + ptySourceDeliveryKey, + samePtySourceDelivery, + type PtySourceDeliveryIdentity, + type PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import { + SshPtySourceAckCoalescer, + type SshPtySourceAckCoalescerOptions +} from './ssh-pty-source-ack-coalescer' +import { + SshPtySourceObligationLedger, + type SshPtySourceAdmissionReservation, + type SshPtySourceConsumerId, + type SshPtySourceObligationState, + type SshPtySourceTokenSnapshot +} from './ssh-pty-source-obligation-ledger' + +export type SshPtySourceObligationTransition = Readonly<{ + identity: PtySourceDeliveryIdentity + spanId: string + consumer: SshPtySourceConsumerId + reason: string +}> + +type TerminalWaiter = { + resolve: () => void + reject: (error: Error) => void +} + +type TerminalWaiterGroup = { + identity: PtySourceDeliveryIdentity + waiters: Set +} + +export class SshPtySourceObligationCoordinator { + private readonly ledger: SshPtySourceObligationLedger + private readonly acknowledgements: SshPtySourceAckCoalescer + private readonly terminalWaiters = new Map() + private disposed = false + + constructor(options: SshPtySourceAckCoalescerOptions) { + this.ledger = new SshPtySourceObligationLedger(options.onTokenClosed) + this.acknowledgements = new SshPtySourceAckCoalescer(options) + } + + open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void { + if (this.disposed) { + throw new Error('SSH PTY source obligation coordinator is disposed') + } + this.ledger.open(identity, checkpointSourceEndSu) + } + + reserve( + identity: PtySourceDeliveryIdentity, + span: PtySourceSpan, + requiredConsumers: readonly SshPtySourceConsumerId[] + ): SshPtySourceAdmissionReservation { + return this.ledger.reserve(identity, span, requiredConsumers) + } + + commit(reservation: SshPtySourceAdmissionReservation): void { + this.ledger.commit(reservation) + } + + rollback(reservation: SshPtySourceAdmissionReservation): boolean { + return this.ledger.rollback(reservation) + } + + rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean { + const rolledBack = this.ledger.rollbackCommitted(reservation) + if (rolledBack) { + this.maybeResolveTerminal(reservation.span) + } + return rolledBack + } + + settle(transition: SshPtySourceObligationTransition): boolean { + this.requireSpanIdentity(transition) + const changed = this.ledger.settle(transition.spanId, transition.consumer, transition.reason) + this.queueEligibleAck(transition.identity) + return changed + } + + beginTransfer(transition: SshPtySourceObligationTransition, to: SshPtySourceConsumerId): boolean { + this.requireSpanIdentity(transition) + return this.ledger.beginTransfer(transition.spanId, transition.consumer, to, transition.reason) + } + + commitTransfer(transition: Omit): boolean { + this.requireSpanIdentity(transition) + const changed = this.ledger.commitTransfer(transition.spanId, transition.consumer) + this.queueEligibleAck(transition.identity) + return changed + } + + cancelTransfer(transition: SshPtySourceObligationTransition): boolean { + this.requireSpanIdentity(transition) + const changed = this.ledger.cancelTransfer( + transition.spanId, + transition.consumer, + transition.reason + ) + this.queueEligibleAck(transition.identity) + return changed + } + + rollbackTransfer(transition: SshPtySourceObligationTransition): boolean { + this.requireSpanIdentity(transition) + return this.ledger.rollbackTransfer(transition.spanId, transition.consumer) + } + + seal(identity: PtySourceDeliveryIdentity): void { + if (this.ledger.snapshot(identity).state === 'sealed-unsettled') { + return + } + this.ledger.seal(identity) + } + + markExitPublished(identity: PtySourceDeliveryIdentity): void { + this.queueEligibleAck(identity) + this.ledger.markExitPublished(identity) + } + + whenTerminal(identity: PtySourceDeliveryIdentity): Promise { + if (this.isTerminal(identity)) { + return Promise.resolve() + } + const key = ptySourceDeliveryKey(identity) + let group = this.terminalWaiters.get(key) + if (!group) { + group = { identity: Object.freeze({ ...identity }), waiters: new Set() } + this.terminalWaiters.set(key, group) + } + return new Promise((resolve, reject) => { + group!.waiters.add({ resolve, reject }) + }) + } + + beginExitTimeout(identity: PtySourceDeliveryIdentity) { + return this.ledger.beginExitTimeout(identity) + } + + applyCancellationProof( + identity: PtySourceDeliveryIdentity, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): void { + const snapshot = this.ledger.snapshot(identity) + if ( + snapshot.state === 'closed' && + snapshot.receivedEndSu === proof.sentEndSu && + snapshot.ackPublishedEndSu === proof.creditedEndSu + ) { + return + } + this.ledger.applyCancellationProof(identity, proof) + this.rejectWaiters( + (group) => samePtySourceDelivery(group.identity, identity), + new Error('ssh_source_delivery_canceled') + ) + } + + applyRecoveryCancellationProof( + identity: PtySourceDeliveryIdentity, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): void { + this.ledger.applyRecoveryCancellationProof(identity, proof) + this.rejectWaiters( + (group) => samePtySourceDelivery(group.identity, identity), + new Error('ssh_source_delivery_canceled') + ) + } + + closeGeneration(providerGeneration: number, reason: string): number { + this.rejectWaiters( + (group) => group.identity.providerGeneration === providerGeneration, + new Error(reason) + ) + const closed = this.ledger.closeGeneration(providerGeneration, reason) + this.acknowledgements.cancelGeneration(providerGeneration, reason) + return closed + } + + snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot { + return this.ledger.snapshot(identity) + } + + modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number { + return this.ledger.modelAcceptedEnd(identity) + } + + obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState { + return this.ledger.obligation(spanId, consumer) + } + + spanIdentity(spanId: string): PtySourceSpan { + return this.ledger.spanIdentity(spanId) + } + + hasRetainedSpan(spanId: string): boolean { + return this.ledger.hasRetainedSpan(spanId) + } + + flushAcknowledgements(): void { + this.acknowledgements.flush() + } + + dispose(reason?: string): void { + if (this.disposed) { + return + } + this.disposed = true + this.rejectWaiters(() => true, new Error(reason ?? 'SSH PTY source obligations disposed')) + this.ledger.closeAll(reason ?? 'SSH PTY source obligation coordinator disposed') + this.acknowledgements.dispose(reason) + } + + private queueEligibleAck(identity: PtySourceDeliveryIdentity): void { + const publication = this.ledger.queueAck(identity) + if (publication) { + this.acknowledgements.enqueue(publication) + } + this.maybeResolveTerminal(identity) + } + + private isTerminal(identity: PtySourceDeliveryIdentity): boolean { + const snapshot = this.ledger.snapshot(identity) + return ( + snapshot.obligationsTerminalEndSu === snapshot.receivedEndSu && + snapshot.ackQueuedEndSu === snapshot.receivedEndSu + ) + } + + private maybeResolveTerminal(identity: PtySourceDeliveryIdentity): void { + if (!this.isTerminal(identity)) { + return + } + const group = this.terminalWaiters.get(ptySourceDeliveryKey(identity)) + if (!group) { + return + } + this.terminalWaiters.delete(ptySourceDeliveryKey(identity)) + for (const waiter of group.waiters) { + waiter.resolve() + } + } + + private rejectWaiters(predicate: (group: TerminalWaiterGroup) => boolean, error: Error): void { + for (const [key, group] of this.terminalWaiters) { + if (!predicate(group)) { + continue + } + this.terminalWaiters.delete(key) + for (const waiter of group.waiters) { + waiter.reject(error) + } + } + } + + private requireSpanIdentity( + transition: Pick + ): void { + if (!samePtySourceDelivery(this.ledger.spanIdentity(transition.spanId), transition.identity)) { + throw new Error('SSH PTY source obligation transition has a stale delivery identity') + } + } +} diff --git a/src/main/ipc/ssh-pty-source-obligation-ledger.test.ts b/src/main/ipc/ssh-pty-source-obligation-ledger.test.ts new file mode 100644 index 00000000000..ec6ad29b41e --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-ledger.test.ts @@ -0,0 +1,305 @@ +import { describe, expect, it } from 'vitest' +import type { + PtySourceDeliveryIdentity, + PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import { SshPtySourceObligationLedger } from './ssh-pty-source-obligation-ledger' + +function identity( + deliveryToken = 'token-1', + overrides: Partial = {} +): PtySourceDeliveryIdentity { + return { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken, + ...overrides + } +} + +function span( + owner: PtySourceDeliveryIdentity, + spanId: string, + sourceStartSu: number, + data: string +): PtySourceSpan { + return Object.freeze({ + ...owner, + spanId, + sourceStartSu, + sourceEndSu: sourceStartSu + data.length, + displayStart: sourceStartSu, + displayEnd: sourceStartSu + data.length, + data, + splittable: true, + transform: Object.freeze({ + transformed: false, + rawLengthSu: data.length, + scalarSafe: true + }) + }) +} + +function commitSpan( + ledger: SshPtySourceObligationLedger, + owner: PtySourceDeliveryIdentity, + sourceSpan: PtySourceSpan, + consumers: ('model' | 'desktop')[] = ['model', 'desktop'] +) { + const reservation = ledger.reserve(owner, sourceSpan, consumers) + ledger.commit(reservation) + return reservation +} + +describe('SshPtySourceObligationLedger', () => { + it('rolls back an uncommitted admission without consuming its source coordinate', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + const first = ledger.reserve(owner, span(owner, 'span-1', 0, 'abc'), ['model']) + + expect(ledger.rollback(first)).toBe(true) + const retry = ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model']) + ledger.commit(retry) + expect(ledger.snapshot(owner).receivedEndSu).toBe(3) + }) + + it('rolls back a committed tail while every obligation is still open', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + const reservation = commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc')) + + expect(ledger.rollbackCommitted(reservation)).toBe(true) + expect(ledger.snapshot(owner)).toMatchObject({ receivedEndSu: 0, openSpans: 0 }) + expect(() => ledger.spanIdentity('span-1')).toThrow('Unknown or reclaimed') + expect( + ledger.commit(ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model'])) + ).toBeUndefined() + }) + + it('keeps terminal, queued, and successfully published ACK ends independent', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd')) + + ledger.settle('span-1', 'model', 'emulator-receipt') + expect(ledger.snapshot(owner)).toMatchObject({ + obligationsTerminalEndSu: 0, + ackQueuedEndSu: 0, + ackPublishedEndSu: 0 + }) + ledger.settle('span-1', 'desktop', 'renderer-parse') + expect(ledger.snapshot(owner)).toMatchObject({ + obligationsTerminalEndSu: 4, + ackQueuedEndSu: 0, + ackPublishedEndSu: 0 + }) + const publication = ledger.queueAck(owner)! + expect(ledger.snapshot(owner)).toMatchObject({ + obligationsTerminalEndSu: 4, + ackQueuedEndSu: 4, + ackPublishedEndSu: 0 + }) + publication.onSettled({ ok: false, error: new Error('write callback failed') }) + expect(ledger.snapshot(owner)).toMatchObject({ ackQueuedEndSu: 4, ackPublishedEndSu: 0 }) + ledger.retryQueuedAck(owner)!.onSettled({ ok: true }) + expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 0 }) + }) + + it('checkpoints only the contiguous model-settled prefix', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd')) + commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh')) + + expect(ledger.modelAcceptedEnd(owner)).toBe(0) + ledger.settle('span-2', 'model', 'out-of-order') + expect(ledger.modelAcceptedEnd(owner)).toBe(0) + ledger.settle('span-1', 'model', 'emulator-receipt') + expect(ledger.modelAcceptedEnd(owner)).toBe(8) + }) + + it('continues checkpoints from the reclaimed ACK-published prefix', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd')) + commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh')) + ledger.settle('span-1', 'model', 'emulator-receipt') + ledger.settle('span-1', 'desktop', 'renderer-parse') + ledger.queueAck(owner)!.onSettled({ ok: true }) + + expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 1 }) + expect(ledger.modelAcceptedEnd(owner)).toBe(4) + ledger.settle('span-2', 'model', 'emulator-receipt') + expect(ledger.modelAcceptedEnd(owner)).toBe(8) + }) + + it('requires an exact transfer fence before a desktop obligation becomes terminal', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc')) + ledger.settle('span-1', 'model', 'emulator-receipt') + + expect(ledger.beginTransfer('span-1', 'desktop', 'model', 'renderer-send-failed')).toBe(true) + expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(0) + expect(ledger.commitTransfer('span-1', 'desktop')).toBe(true) + expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(3) + expect(ledger.obligation('span-1', 'desktop')).toMatchObject({ + state: 'transferred', + to: 'model' + }) + }) + + it('keeps sealed exit state until final ACK publication succeeds', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model']) + ledger.seal(owner) + expect(() => ledger.markExitPublished(owner)).toThrow('terminal ACK') + ledger.settle('tail', 'model', 'emulator-receipt') + const publication = ledger.queueAck(owner)! + ledger.markExitPublished(owner) + expect(ledger.snapshot(owner).state).toBe('sealed-unsettled') + publication.onSettled({ ok: true }) + expect(ledger.snapshot(owner).state).toBe('closed') + }) + + it('publishes token cancellation intent before accepting timeout cleanup proof', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model']) + ledger.seal(owner) + + expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })).toThrow() + expect(ledger.beginExitTimeout(owner)).toEqual({ + id: owner.id, + deliveryToken: owner.deliveryToken, + clientGeneration: owner.clientGeneration, + ownerGeneration: owner.ownerGeneration + }) + ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 }) + expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 }) + }) + + it('ignores a late successful write callback after generation-close proof', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'), ['model']) + ledger.settle('span-1', 'model', 'accepted') + const publication = ledger.queueAck(owner)! + + expect(ledger.closeGeneration(1, 'provider-closed')).toBe(1) + publication.onSettled({ ok: true }) + expect(ledger.snapshot(owner)).toMatchObject({ + state: 'closed', + generationClosed: true, + ackPublishedEndSu: 0 + }) + expect(() => ledger.obligation('span-1', 'model')).toThrow('reclaimed') + }) + + it('rejects stale generations, tokens, and non-contiguous source spans', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + expect(() => + ledger.reserve(identity('stale'), span(identity('stale'), 'stale', 0, 'x'), ['model']) + ).toThrow('stale') + expect(() => ledger.reserve(owner, span(owner, 'gap', 1, 'x'), ['model'])).toThrow( + 'non-contiguous' + ) + }) + + it('requires cancellation proof to match the exact local received and published ends', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model']) + ledger.seal(owner) + ledger.beginExitTimeout(owner) + + expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 3, creditedEndSu: 0 })).toThrow( + 'invalid' + ) + expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 1 })).toThrow( + 'invalid' + ) + }) + + it('applies recovery cancellation proof over a locally admitted prefix', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner, 4) + commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model']) + + ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 12, creditedEndSu: 4 }) + + expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 }) + }) + + it('rejects recovery cancellation proof that misses local intake state', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner, 4) + commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model']) + + expect(() => + ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 7, creditedEndSu: 4 }) + ).toThrow('invalid') + expect(() => + ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 8, creditedEndSu: 5 }) + ).toThrow('invalid') + }) + + it('bounds closed-token tombstones and removes committed reservation indexes', () => { + const ledger = new SshPtySourceObligationLedger() + const owners = Array.from({ length: 300 }, (_, index) => + identity(`token-${index}`, { + id: `pty-${index}`, + ptyIncarnation: `incarnation-${index}` + }) + ) + for (const [index, owner] of owners.entries()) { + ledger.open(owner) + commitSpan(ledger, owner, span(owner, `span-${index}`, 0, 'x'), ['model']) + ledger.closeGeneration(1, 'generation-closed') + } + + expect(() => ledger.snapshot(owners[0])).toThrow('stale') + expect(ledger.snapshot(owners.at(-1)!)).toMatchObject({ state: 'closed' }) + }) + + it('reclaims uncommitted reservations on exact cancellation proof', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + const reservation = ledger.reserve(owner, span(owner, 'pending', 0, 'x'), ['model']) + ledger.seal(owner) + ledger.beginExitTimeout(owner) + + ledger.applyCancellationProof(owner, { sentEndSu: 0, creditedEndSu: 0 }) + + expect(ledger.rollback(reservation)).toBe(false) + expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 }) + }) + + it('rejects reopening a recently closed one-use token', () => { + const ledger = new SshPtySourceObligationLedger() + const owner = identity() + ledger.open(owner) + ledger.closeGeneration(owner.providerGeneration, 'closed') + + expect(() => ledger.open(owner)).toThrow('already used') + }) +}) diff --git a/src/main/ipc/ssh-pty-source-obligation-ledger.ts b/src/main/ipc/ssh-pty-source-obligation-ledger.ts new file mode 100644 index 00000000000..976f7d2fc5c --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-ledger.ts @@ -0,0 +1,329 @@ +import { + ptySourceDeliveryKey, + samePtySourceDelivery, + type PtySourceDeliveryIdentity, + type PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import { + assertNonNegativeSafeInteger, + assertPtySourceIdentity, + assertPtySourceSpan +} from '../../shared/pty-source-credit-validation' +import type { + SshPtySourceAckPublication, + SshPtySourceAdmissionReservation, + SshPtySourceConsumerId, + SshPtySourceObligationState, + SshPtySourceTokenSnapshot +} from './ssh-pty-source-obligation-contract' +import { createSshPtySourceAckPublication } from './ssh-pty-source-ack-publication' +import { + beginSourceExitTimeout, + cancelOpenSourceObligations, + closeAllSourceTokens, + closeSourceGeneration, + closeSourceToken, + createSourceSpanRecord, + createSourceToken, + markSourceExitPublished, + requireSourceSpan, + rollbackCommittedSourceSpan, + sealSourceToken, + snapshotSourceToken, + type ReservationRecord, + type TokenRecord +} from './ssh-pty-source-obligation-state' +import { + applySourceRecoveryCancellationProof, + cancelSourceObligationTransfer, + commitSourceObligationTransfer, + modelAcceptedSourceEnd, + rollbackSourceObligationTransfer, + transitionOpenSourceObligation +} from './ssh-pty-source-obligation-transitions' + +export type { + SshPtySourceAckPublication, + SshPtySourceAdmissionReservation, + SshPtySourceConsumerId, + SshPtySourceObligationState, + SshPtySourceTokenSnapshot +} from './ssh-pty-source-obligation-contract' + +export class SshPtySourceObligationLedger { + private readonly tokens = new Map() + private readonly closedSnapshots = new Map() + private readonly reservations = new Map() + private readonly spanOwners = new Map() + private nextReservationId = 1 + + constructor( + private readonly onTokenClosed: (identity: PtySourceDeliveryIdentity) => void = () => {} + ) {} + + open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void { + assertPtySourceIdentity(identity) + assertNonNegativeSafeInteger(checkpointSourceEndSu, 'checkpointSourceEndSu') + const key = ptySourceDeliveryKey(identity) + if (this.tokens.has(key) || this.closedSnapshots.has(key)) { + throw new Error('SSH PTY source token was already used') + } + this.tokens.set(key, createSourceToken(identity, checkpointSourceEndSu)) + } + + reserve( + identity: PtySourceDeliveryIdentity, + span: PtySourceSpan, + requiredConsumers: readonly SshPtySourceConsumerId[] + ): SshPtySourceAdmissionReservation { + const token = this.requireToken(identity) + if (token.state !== 'active') { + throw new Error('SSH PTY source token no longer admits data') + } + assertPtySourceSpan(span) + if ( + !samePtySourceDelivery(token.identity, span) || + span.sourceStartSu !== token.receivedEndSu || + this.spanOwners.has(span.spanId) + ) { + throw new Error('SSH PTY source span is stale, duplicate, or non-contiguous') + } + const uniqueConsumers = Array.from(new Set(requiredConsumers)) + if (!uniqueConsumers.includes('model')) { + throw new Error('SSH PTY source span requires the terminal model obligation') + } + const reservation = Object.freeze({ + reservationId: `ssh-source-admission:${this.nextReservationId++}`, + span, + requiredConsumers: Object.freeze(uniqueConsumers) + }) + this.reservations.set(reservation.reservationId, { + reservation, + state: 'reserved' + }) + return reservation + } + + commit(reservation: SshPtySourceAdmissionReservation): void { + const record = this.requireReservation(reservation) + if (record.state !== 'reserved') { + throw new Error('SSH PTY source admission reservation is not pending') + } + const token = this.requireToken(reservation.span) + if (token.state !== 'active' || token.receivedEndSu !== reservation.span.sourceStartSu) { + throw new Error('SSH PTY source admission reservation became stale') + } + const spanRecord = createSourceSpanRecord(reservation.span, reservation.requiredConsumers) + token.spans.push(spanRecord) + token.receivedEndSu = reservation.span.sourceEndSu + this.spanOwners.set(reservation.span.spanId, token) + this.reservations.delete(reservation.reservationId) + } + + rollback(reservation: SshPtySourceAdmissionReservation): boolean { + const record = this.reservations.get(reservation.reservationId) + if (!record || record.reservation !== reservation || record.state !== 'reserved') { + return false + } + this.reservations.delete(reservation.reservationId) + return true + } + + rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean { + const token = this.tokens.get(ptySourceDeliveryKey(reservation.span)) + if (!token || !samePtySourceDelivery(token.identity, reservation.span)) { + return false + } + return rollbackCommittedSourceSpan(token, reservation, this.spanOwners) + } + + settle(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean { + return this.transitionOpen(spanId, consumer, Object.freeze({ state: 'settled', reason })) + } + + beginTransfer( + spanId: string, + consumer: SshPtySourceConsumerId, + to: SshPtySourceConsumerId, + reason: string + ): boolean { + return this.transitionOpen( + spanId, + consumer, + Object.freeze({ state: 'transferring', to, reason }) + ) + } + + commitTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean { + return commitSourceObligationTransfer(this.spanOwners, spanId, consumer) + } + + cancelTransfer(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean { + return cancelSourceObligationTransfer(this.spanOwners, spanId, consumer, reason) + } + + rollbackTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean { + return rollbackSourceObligationTransfer(this.spanOwners, spanId, consumer) + } + + queueAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null { + const token = this.requireToken(identity) + if (token.obligationsTerminalEndSu <= token.ackQueuedEndSu) { + return null + } + const endSu = token.obligationsTerminalEndSu + token.ackQueuedEndSu = endSu + return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () => + this.maybeClose(token) + ) + } + + retryQueuedAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null { + const token = this.requireToken(identity) + if (token.ackQueuedEndSu <= token.ackPublishedEndSu) { + return null + } + const endSu = token.ackQueuedEndSu + return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () => + this.maybeClose(token) + ) + } + + seal(identity: PtySourceDeliveryIdentity): void { + sealSourceToken(this.requireToken(identity)) + } + + markExitPublished(identity: PtySourceDeliveryIdentity): void { + const token = this.requireToken(identity) + markSourceExitPublished(token) + this.maybeClose(token) + } + + beginExitTimeout(identity: PtySourceDeliveryIdentity): Readonly<{ + id: string + deliveryToken: string + clientGeneration: number + ownerGeneration: number + }> { + return beginSourceExitTimeout(this.requireToken(identity)) + } + + applyCancellationProof( + identity: PtySourceDeliveryIdentity, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): void { + const token = this.requireToken(identity) + if ( + token.state !== 'canceling' || + proof.sentEndSu !== token.receivedEndSu || + proof.creditedEndSu !== token.ackPublishedEndSu + ) { + throw new Error('SSH PTY source cancellation proof is stale or invalid') + } + cancelOpenSourceObligations(token, 'relay-cancellation-proof') + this.closeToken(token) + } + + applyRecoveryCancellationProof( + identity: PtySourceDeliveryIdentity, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> + ): void { + const token = this.requireToken(identity) + applySourceRecoveryCancellationProof(token, proof) + this.closeToken(token) + } + + closeGeneration(providerGeneration: number, reason: string): number { + return closeSourceGeneration( + this.tokens, + this.reservations, + providerGeneration, + reason, + (token) => this.closeToken(token) + ) + } + + closeAll(reason: string): number { + return closeAllSourceTokens(this.tokens, this.reservations, reason, (token) => + this.closeToken(token) + ) + } + + snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot { + const key = ptySourceDeliveryKey(identity) + const token = this.tokens.get(key) + if (token && samePtySourceDelivery(token.identity, identity)) { + return snapshotSourceToken(token) + } + const closed = this.closedSnapshots.get(key) + if (closed && samePtySourceDelivery(closed, identity)) { + return closed + } + throw new Error('Unknown or stale SSH PTY source token') + } + + modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number { + return modelAcceptedSourceEnd(this.requireToken(identity)) + } + + obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState { + const obligation = requireSourceSpan(this.spanOwners, spanId).span.obligations.get(consumer) + if (!obligation) { + throw new Error('SSH PTY source consumer obligation does not exist') + } + return obligation + } + + spanIdentity(spanId: string): PtySourceSpan { + return requireSourceSpan(this.spanOwners, spanId).span.span + } + + hasRetainedSpan(spanId: string): boolean { + return this.spanOwners.has(spanId) + } + + private transitionOpen( + spanId: string, + consumer: SshPtySourceConsumerId, + next: SshPtySourceObligationState + ): boolean { + return transitionOpenSourceObligation(this.spanOwners, spanId, consumer, next) + } + + private maybeClose(token: TokenRecord): void { + if ( + token.state === 'sealed-unsettled' && + token.exitPublished && + token.ackPublishedEndSu === token.receivedEndSu + ) { + this.closeToken(token) + } + } + + private closeToken(token: TokenRecord): void { + closeSourceToken( + token, + this.tokens, + this.closedSnapshots, + this.reservations, + this.spanOwners, + this.onTokenClosed + ) + } + + private requireReservation(reservation: SshPtySourceAdmissionReservation): ReservationRecord { + const record = this.reservations.get(reservation.reservationId) + if (!record || record.reservation !== reservation) { + throw new Error('Unknown SSH PTY source admission reservation') + } + return record + } + + private requireToken(identity: PtySourceDeliveryIdentity): TokenRecord { + const token = this.tokens.get(ptySourceDeliveryKey(identity)) + if (!token || !samePtySourceDelivery(token.identity, identity)) { + throw new Error('Unknown or stale SSH PTY source token') + } + return token + } +} diff --git a/src/main/ipc/ssh-pty-source-obligation-state.ts b/src/main/ipc/ssh-pty-source-obligation-state.ts new file mode 100644 index 00000000000..a51694518a4 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-state.ts @@ -0,0 +1,287 @@ +import type { + PtySourceDeliveryIdentity, + PtySourceSpan +} from '../../shared/pty-source-credit-contract' +import { + ptySourceDeliveryKey, + samePtySourceDelivery +} from '../../shared/pty-source-credit-contract' +import type { + SshPtySourceAdmissionReservation, + SshPtySourceConsumerId, + SshPtySourceObligationState, + SshPtySourceTokenSnapshot +} from './ssh-pty-source-obligation-contract' + +export const CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT = 256 + +export type SpanRecord = { + span: PtySourceSpan + obligations: Map +} + +export type ReservationRecord = { + reservation: SshPtySourceAdmissionReservation + state: 'reserved' | 'committed' | 'rolled-back' +} + +export type TokenRecord = { + identity: PtySourceDeliveryIdentity + state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed' + checkpointSourceEndSu: number + receivedEndSu: number + obligationsTerminalEndSu: number + ackQueuedEndSu: number + ackPublishedEndSu: number + spans: SpanRecord[] + exitPublished: boolean + generationClosed: boolean +} + +export function createSourceToken( + identity: PtySourceDeliveryIdentity, + checkpointSourceEndSu: number +): TokenRecord { + return { + identity: Object.freeze({ ...identity }), + state: 'active', + checkpointSourceEndSu, + receivedEndSu: checkpointSourceEndSu, + obligationsTerminalEndSu: checkpointSourceEndSu, + ackQueuedEndSu: checkpointSourceEndSu, + ackPublishedEndSu: checkpointSourceEndSu, + spans: [], + exitPublished: false, + generationClosed: false + } +} + +export function createSourceSpanRecord( + span: PtySourceSpan, + consumers: readonly SshPtySourceConsumerId[] +): SpanRecord { + return { + span, + obligations: new Map( + consumers.map((consumer) => [consumer, Object.freeze({ state: 'open' as const })]) + ) + } +} + +export function sealSourceToken(token: TokenRecord): void { + if (token.state !== 'active') { + throw new Error('SSH PTY source token cannot be sealed from its current state') + } + token.state = 'sealed-unsettled' +} + +export function markSourceExitPublished(token: TokenRecord): void { + if (token.state !== 'sealed-unsettled') { + throw new Error('SSH PTY source exit publication requires a sealed token') + } + if ( + token.obligationsTerminalEndSu !== token.receivedEndSu || + token.ackQueuedEndSu !== token.receivedEndSu + ) { + throw new Error('SSH PTY source exit cannot publish before terminal ACK queueing') + } + token.exitPublished = true +} + +export function beginSourceExitTimeout(token: TokenRecord): Readonly<{ + id: string + deliveryToken: string + clientGeneration: number + ownerGeneration: number +}> { + if (token.state !== 'sealed-unsettled') { + throw new Error('SSH PTY source exit timeout requires a sealed token') + } + token.state = 'canceling' + return Object.freeze({ + id: token.identity.id, + deliveryToken: token.identity.deliveryToken, + clientGeneration: token.identity.clientGeneration, + ownerGeneration: token.identity.ownerGeneration + }) +} + +function obligationIsTerminal(obligation: SshPtySourceObligationState): boolean { + return ( + obligation.state === 'settled' || + obligation.state === 'transferred' || + obligation.state === 'canceled' + ) +} + +export function snapshotSourceToken(token: TokenRecord): SshPtySourceTokenSnapshot { + return Object.freeze({ + ...token.identity, + state: token.state, + receivedEndSu: token.receivedEndSu, + obligationsTerminalEndSu: token.obligationsTerminalEndSu, + ackQueuedEndSu: token.ackQueuedEndSu, + ackPublishedEndSu: token.ackPublishedEndSu, + openSpans: token.spans.length, + exitPublished: token.exitPublished, + generationClosed: token.generationClosed + }) +} + +export function advanceSourceTerminalEnd(token: TokenRecord): void { + let endSu = token.obligationsTerminalEndSu + for (const record of token.spans) { + if (record.span.sourceEndSu <= endSu) { + continue + } + if ( + record.span.sourceStartSu !== endSu || + !Array.from(record.obligations.values()).every(obligationIsTerminal) + ) { + break + } + endSu = record.span.sourceEndSu + } + token.obligationsTerminalEndSu = endSu +} + +export function cancelOpenSourceObligations(token: TokenRecord, reason: string): void { + for (const record of token.spans) { + for (const [consumer, obligation] of record.obligations) { + if (obligation.state === 'open' || obligation.state === 'transferring') { + record.obligations.set(consumer, Object.freeze({ state: 'canceled', reason })) + } + } + } + advanceSourceTerminalEnd(token) +} + +export function reclaimPublishedSourcePrefix( + token: TokenRecord, + spanOwners: Map +): void { + while (token.spans[0]?.span.sourceEndSu <= token.ackPublishedEndSu) { + const record = token.spans.shift()! + spanOwners.delete(record.span.spanId) + } +} + +export function releaseSourceTokenSpans( + token: TokenRecord, + spanOwners: Map +): void { + for (const record of token.spans) { + spanOwners.delete(record.span.spanId) + } + token.spans = [] +} + +export function releaseSourceTokenReservations( + token: TokenRecord, + reservations: Map +): void { + for (const [reservationId, record] of reservations) { + if (samePtySourceDelivery(record.reservation.span, token.identity)) { + record.state = 'rolled-back' + reservations.delete(reservationId) + } + } +} + +export function rollbackCommittedSourceSpan( + token: TokenRecord, + reservation: SshPtySourceAdmissionReservation, + spanOwners: Map +): boolean { + const last = token.spans.at(-1) + if ( + last?.span !== reservation.span || + token.receivedEndSu !== reservation.span.sourceEndSu || + token.obligationsTerminalEndSu > reservation.span.sourceStartSu || + token.ackQueuedEndSu > reservation.span.sourceStartSu || + Array.from(last.obligations.values()).some((obligation) => obligation.state !== 'open') + ) { + return false + } + token.spans.pop() + token.receivedEndSu = reservation.span.sourceStartSu + spanOwners.delete(reservation.span.spanId) + return true +} + +export function requireSourceSpan( + spanOwners: ReadonlyMap, + spanId: string +): { token: TokenRecord; span: SpanRecord } { + const token = spanOwners.get(spanId) + const span = token?.spans.find((candidate) => candidate.span.spanId === spanId) + if (!token || !span) { + throw new Error('Unknown or reclaimed SSH PTY source span') + } + return { token, span } +} + +export function closeSourceGeneration( + tokens: Map, + reservations: Map, + providerGeneration: number, + reason: string, + closeToken: (token: TokenRecord) => void +): number { + let closed = 0 + for (const token of Array.from(tokens.values())) { + if (token.identity.providerGeneration !== providerGeneration || token.state === 'closed') { + continue + } + token.generationClosed = true + cancelOpenSourceObligations(token, reason) + closeToken(token) + closed++ + } + for (const [id, record] of reservations) { + if ( + record.state === 'reserved' && + record.reservation.span.providerGeneration === providerGeneration + ) { + record.state = 'rolled-back' + reservations.delete(id) + } + } + return closed +} + +export function closeAllSourceTokens( + tokens: Map, + reservations: Map, + reason: string, + closeToken: (token: TokenRecord) => void +): number { + let closed = 0 + const generations = new Set( + Array.from(tokens.values(), (token) => token.identity.providerGeneration) + ) + for (const providerGeneration of generations) { + closed += closeSourceGeneration(tokens, reservations, providerGeneration, reason, closeToken) + } + return closed +} + +export function closeSourceToken( + token: TokenRecord, + tokens: Map, + closedSnapshots: Map, + reservations: Map, + spanOwners: Map, + onTokenClosed: (identity: PtySourceDeliveryIdentity) => void +): void { + token.state = 'closed' + releaseSourceTokenSpans(token, spanOwners) + releaseSourceTokenReservations(token, reservations) + const key = ptySourceDeliveryKey(token.identity) + tokens.delete(key) + closedSnapshots.set(key, snapshotSourceToken(token)) + while (closedSnapshots.size > CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT) { + closedSnapshots.delete(closedSnapshots.keys().next().value!) + } + onTokenClosed(token.identity) +} diff --git a/src/main/ipc/ssh-pty-source-obligation-transitions.ts b/src/main/ipc/ssh-pty-source-obligation-transitions.ts new file mode 100644 index 00000000000..692a891be48 --- /dev/null +++ b/src/main/ipc/ssh-pty-source-obligation-transitions.ts @@ -0,0 +1,100 @@ +import type { + SshPtySourceConsumerId, + SshPtySourceObligationState +} from './ssh-pty-source-obligation-contract' +import { + advanceSourceTerminalEnd, + cancelOpenSourceObligations, + requireSourceSpan, + type TokenRecord +} from './ssh-pty-source-obligation-state' + +export function applySourceRecoveryCancellationProof( + token: TokenRecord, + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> +): void { + if ( + token.state !== 'active' || + proof.sentEndSu < token.receivedEndSu || + proof.creditedEndSu !== token.ackPublishedEndSu || + proof.creditedEndSu > token.receivedEndSu + ) { + throw new Error('SSH PTY source recovery cancellation proof is stale or invalid') + } + cancelOpenSourceObligations(token, 'relay-recovery-cancellation-proof') +} + +export function transitionOpenSourceObligation( + spanOwners: ReadonlyMap, + spanId: string, + consumer: SshPtySourceConsumerId, + next: SshPtySourceObligationState +): boolean { + const { token, span } = requireSourceSpan(spanOwners, spanId) + if (span.obligations.get(consumer)?.state !== 'open') { + return false + } + span.obligations.set(consumer, next) + advanceSourceTerminalEnd(token) + return true +} + +export function commitSourceObligationTransfer( + spanOwners: ReadonlyMap, + spanId: string, + consumer: SshPtySourceConsumerId +): boolean { + const { token, span } = requireSourceSpan(spanOwners, spanId) + const current = span.obligations.get(consumer) + if (current?.state !== 'transferring') { + return false + } + span.obligations.set( + consumer, + Object.freeze({ state: 'transferred', to: current.to, reason: current.reason }) + ) + advanceSourceTerminalEnd(token) + return true +} + +export function cancelSourceObligationTransfer( + spanOwners: ReadonlyMap, + spanId: string, + consumer: SshPtySourceConsumerId, + reason: string +): boolean { + const { token, span } = requireSourceSpan(spanOwners, spanId) + if (span.obligations.get(consumer)?.state !== 'transferring') { + return false + } + span.obligations.set(consumer, Object.freeze({ state: 'canceled', reason })) + advanceSourceTerminalEnd(token) + return true +} + +export function rollbackSourceObligationTransfer( + spanOwners: ReadonlyMap, + spanId: string, + consumer: SshPtySourceConsumerId +): boolean { + const { span } = requireSourceSpan(spanOwners, spanId) + if (span.obligations.get(consumer)?.state !== 'transferring') { + return false + } + span.obligations.set(consumer, Object.freeze({ state: 'open' })) + return true +} + +export function modelAcceptedSourceEnd(token: TokenRecord): number { + let acceptedEndSu = token.ackPublishedEndSu + for (const record of token.spans) { + if ( + record.span.sourceStartSu !== acceptedEndSu || + record.obligations.get('model')?.state !== 'settled' + ) { + break + } + acceptedEndSu = record.span.sourceEndSu + } + return acceptedEndSu +} diff --git a/src/main/ipc/ssh.test.ts b/src/main/ipc/ssh.test.ts index a02e36ca7eb..05de50fb2d4 100644 --- a/src/main/ipc/ssh.test.ts +++ b/src/main/ipc/ssh.test.ts @@ -11,6 +11,8 @@ const { mockConnectionManager, mockDeployAndLaunchRelay, mockForceStopRelayForTarget, + mockAcceptSshPtyOutputData, + mockAcceptSshPtyOutputExit, mockMux, mockPtyProvider, mockFsProvider, @@ -49,10 +51,13 @@ const { }, mockDeployAndLaunchRelay: vi.fn(), mockForceStopRelayForTarget: vi.fn(), + mockAcceptSshPtyOutputData: vi.fn().mockResolvedValue({}), + mockAcceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined), mockMux: { dispose: vi.fn(), isDisposed: vi.fn().mockReturnValue(false), onNotification: vi.fn(), + onNotificationByMethod: vi.fn().mockReturnValue(() => {}), onRequest: vi.fn().mockReturnValue(() => {}), onDispose: vi.fn().mockReturnValue(() => {}), request: vi.fn().mockResolvedValue({}), @@ -65,7 +70,8 @@ const { onReplay: vi.fn(), attach: vi.fn(), attachForReconnect: vi.fn().mockResolvedValue({}), - shutdown: vi.fn() + shutdown: vi.fn(), + providerGeneration: 0 }, mockFsProvider: {}, mockGitProvider: {}, @@ -100,6 +106,25 @@ vi.mock('electron', () => ({ } })) +vi.mock('./ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: mockAcceptSshPtyOutputData, + acceptSshPtyOutputExit: mockAcceptSshPtyOutputExit, + allocateSshPtyProviderGeneration: (() => { + let generation = 0 + return () => ++generation + })(), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + applySshPtySourceCancellationProof: vi.fn().mockReturnValue(false), + applySshPtySourceRecoveryCancellationProof: vi.fn().mockReturnValue(false), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn().mockReturnValue([]), + installSshPtySourceAckPublisher: vi.fn().mockReturnValue(() => {}), + installSshPtySourceCancellationPublisher: vi.fn().mockReturnValue(() => {}) +})) + vi.mock('../ssh/ssh-connection-store', () => ({ SshConnectionStore: class MockSshConnectionStore { constructor() { @@ -142,7 +167,8 @@ vi.mock('../providers/ssh-pty-provider', () => ({ isSshPtyNotFoundError: (err: unknown) => (err instanceof Error ? err.message : String(err)).includes('not found'), SshPtyProvider: class MockSshPtyProvider { - constructor() { + constructor(_targetId: unknown, _mux: unknown, _env: unknown, providerGeneration: number) { + mockPtyProvider.providerGeneration = providerGeneration return mockPtyProvider } } @@ -222,6 +248,9 @@ import { type SshConnectionState, type SshTarget } from '../../shared/ssh-types' +import { PTY_CONSUMER_SESSION_PROTOCOL_VERSION } from '../../shared/pty-consumer-session' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract' import { clearProviderPtyState, deletePtyOwnership, @@ -231,6 +260,16 @@ import { import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation' describe('SSH IPC handlers', () => { + const relayBuildId = '0.1.0+ipc-test' + const ipcTestSource = { + relayPtyId: 'remote-pty', + spanId: 'ipc-test-delivery:0:5', + clientGeneration: 1, + ownerGeneration: 1, + deliveryToken: 'ipc-test-delivery', + sourceStartSu: 0, + sourceEndSu: 5 + } as const const handlers = new Map unknown>() const mockStore = { getRepos: () => [], @@ -272,7 +311,8 @@ describe('SSH IPC handlers', () => { const relayLostStabilizedMs = 5_000 const createRelayLaunchResult = () => ({ transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, - platform: 'linux-x64' + platform: 'linux-x64', + serverBuildId: relayBuildId }) const getLatestRelayDisposeCallback = (): RelayDisposeCallback => { const calls = mockMux.onDispose.mock.calls @@ -322,21 +362,43 @@ describe('SSH IPC handlers', () => { mockConnectionManager.setCallbacks.mockReset() mockConnectionManager.callbacksRef.current = null mockForceStopRelayForTarget.mockReset().mockResolvedValue(undefined) + mockAcceptSshPtyOutputData.mockReset().mockResolvedValue({}) + mockAcceptSshPtyOutputExit.mockReset().mockResolvedValue(undefined) mockDeployAndLaunchRelay.mockReset().mockResolvedValue({ transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, - platform: 'linux-x64' + platform: 'linux-x64', + serverBuildId: relayBuildId }) mockMux.dispose.mockReset() mockMux.isDisposed.mockReset().mockReturnValue(false) mockMux.onNotification.mockReset() + mockMux.onNotificationByMethod.mockReset().mockReturnValue(() => {}) mockMux.onDispose.mockReset().mockReturnValue(() => {}) + mockMux.request.mockReset().mockImplementation((method: string) => + Promise.resolve( + method === 'pty.openClient' + ? { + protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + serverBuildId: relayBuildId, + clientGeneration: 1, + role: 'session-owner', + ownerGeneration: 1, + ownerLease: 'ipc-test-owner', + capabilities: { + outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } + } + } + : {} + ) + ) mockMux.probeLiveness.mockReset().mockResolvedValue(false) mockPtyProvider.onData.mockReset() mockPtyProvider.onExit.mockReset() mockPtyProvider.onReplay.mockReset() mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({}) mockPtyProvider.shutdown.mockReset() + mockPtyProvider.providerGeneration = 0 mockRegisterSshGitProvider.mockReset() mockPortForwardManager.addForward.mockReset() mockPortForwardManager.updateForward.mockReset() @@ -565,6 +627,7 @@ describe('SSH IPC handlers', () => { } mockDeployAndLaunchRelay.mockResolvedValueOnce({ transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, + serverBuildId: relayBuildId, hostPlatform }) mockSshStore.getTarget.mockReturnValue(target) @@ -1050,7 +1113,7 @@ describe('SSH IPC handlers', () => { } }) - it('forwards remote PTY events into the runtime', async () => { + it('forwards remote PTY events through the output intake authority', async () => { const runtime = { onPtyData: vi.fn(), onPtyExit: vi.fn() @@ -1073,24 +1136,47 @@ describe('SSH IPC handlers', () => { }) await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) - const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as - | ((payload: { id: string; data: string }) => void) - | undefined + const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as - | ((payload: { id: string; code: number }) => void) + | ((payload: { + id: string + code: number + providerGeneration: number + ptyIncarnation: string + }) => void) | undefined - onData?.({ id: 'remote-pty', data: 'hello' }) - onExit?.({ id: 'remote-pty', code: 7 }) + onData?.({ + id: 'remote-pty', + data: 'hello', + providerGeneration: mockPtyProvider.providerGeneration, + ptyIncarnation: 'ipc-test-pty', + source: ipcTestSource + }) + onExit?.({ + id: 'remote-pty', + code: 7, + providerGeneration: mockPtyProvider.providerGeneration, + ptyIncarnation: 'ipc-test-pty' + }) - expect(runtime.onPtyData).toHaveBeenCalledWith( - 'remote-pty', - 'hello', - expect.any(Number), - 'hello'.length, - undefined - ) - expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 7, undefined) + expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith({ + id: 'remote-pty', + data: 'hello', + providerGeneration: mockPtyProvider.providerGeneration, + ptyIncarnation: 'ipc-test-pty', + rawLength: 'hello'.length, + transformed: false, + source: ipcTestSource + }) + expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith({ + id: 'remote-pty', + code: 7, + providerGeneration: mockPtyProvider.providerGeneration, + ptyIncarnation: 'ipc-test-pty' + }) + expect(runtime.onPtyData).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() }) it('mirrors SSH state broadcasts onto the runtime client-event stream', async () => { @@ -1380,7 +1466,7 @@ describe('SSH IPC handlers', () => { expect(replacementConnectionManager.disconnect).not.toHaveBeenCalled() }) - it('refreshes live session callbacks to the newest window, store, and runtime', async () => { + it('refreshes live session callbacks to the newest window and output authorities', async () => { const firstWindow = createMockWindow() const secondWindow = createMockWindow() const firstRuntime = { @@ -1411,11 +1497,14 @@ describe('SSH IPC handlers', () => { }) await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' }) - const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as - | ((payload: { id: string; data: string }) => void) - | undefined + const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as - | ((payload: { id: string; code: number }) => void) + | ((payload: { + id: string + code: number + providerGeneration: number + ptyIncarnation: string + }) => void) | undefined const onDetectedPorts = mockPortScannerCallbacks.get('ssh-1') as | ((targetId: string, ports: unknown[], platform: string) => void) @@ -1434,8 +1523,19 @@ describe('SSH IPC handlers', () => { error: 'network down', reconnectAttempt: 0 }) - onData?.({ id: 'remote-pty', data: 'hello' }) - onExit?.({ id: 'remote-pty', code: 9 }) + onData?.({ + id: 'remote-pty', + data: 'hello', + providerGeneration: mockPtyProvider.providerGeneration, + ptyIncarnation: 'ipc-test-pty', + source: ipcTestSource + }) + onExit?.({ + id: 'remote-pty', + code: 9, + providerGeneration: mockPtyProvider.providerGeneration, + ptyIncarnation: 'ipc-test-pty' + }) onDetectedPorts?.( 'ssh-1', [{ host: '127.0.0.1', port: 3000, pid: 12, processName: 'node' }], @@ -1454,33 +1554,20 @@ describe('SSH IPC handlers', () => { connectionGeneration: 1 } }) - expect(secondWindow.webContents.send).toHaveBeenCalledWith( - 'pty:data', + expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith( expect.objectContaining({ id: 'remote-pty', data: 'hello' }) ) - expect(secondWindow.webContents.send).toHaveBeenCalledWith('pty:exit', { - id: 'remote-pty', - code: 9 - }) + expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith( + expect.objectContaining({ id: 'remote-pty', code: 9 }) + ) expect(secondWindow.webContents.send).toHaveBeenCalledWith('ssh:detected-ports-changed', { targetId: 'ssh-1', ports: expect.arrayContaining([expect.objectContaining({ port: 3000 })]) }) - expect(secondRuntime.onPtyData).toHaveBeenCalledWith( - 'remote-pty', - 'hello', - expect.any(Number), - 'hello'.length, - undefined - ) - expect(secondRuntime.onPtyExit).toHaveBeenCalledWith('remote-pty', 9, undefined) + expect(secondRuntime.onPtyData).not.toHaveBeenCalled() + expect(secondRuntime.onPtyExit).not.toHaveBeenCalled() expect(firstRuntime.onPtyData).not.toHaveBeenCalled() expect(firstRuntime.onPtyExit).not.toHaveBeenCalled() - expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( - 'ssh-1', - 'remote-pty', - 'terminated' - ) }) it('re-registers without replacing managers when no targets are connected', () => { diff --git a/src/main/persistence.test.ts b/src/main/persistence.test.ts index 0730f3a98ba..6c98c73a27d 100644 --- a/src/main/persistence.test.ts +++ b/src/main/persistence.test.ts @@ -1548,6 +1548,26 @@ describe('Store', () => { expect(updatedTarget).not.toHaveProperty('systemSshConnectionReuse') }) + it('drops retired per-target SSH terminal source-credit selections', async () => { + const store = await createStore() + store.addSshTarget({ + id: 'ssh-source-credit-on', + label: 'Noisy build host', + host: 'build.example.com', + port: 22, + username: 'dev', + experimentalPtySourceCreditV1: true + } as never) + + expect(store.getSshTarget('ssh-source-credit-on')).not.toHaveProperty( + 'experimentalPtySourceCreditV1' + ) + store.flush() + const persisted = readDataFile() as { sshTargets?: Record[] } + const target = persisted.sshTargets?.find((entry) => entry.id === 'ssh-source-credit-on') + expect(target).not.toHaveProperty('experimentalPtySourceCreditV1') + }) + it('upserts ~/.ssh/config through the real store: rotated port updates in place and persists', async () => { loadUserSshConfigMock.mockReturnValue([{ host: 'cluster' }]) const candidate = (port: number, id: string) => [ @@ -1563,7 +1583,6 @@ describe('Store', () => { expect(inserted).toHaveLength(1) expect(inserted[0]?.source).toBe('ssh-config') expect(inserted[0]?.port).toBe(2200) - // Rotated port: upsert updates the same target in place and normalizeSshTarget must keep `source` (no false re-derive into a permanently-dirty state). sshConfigHostsToTargetsMock.mockReturnValue(candidate(2222, 'ssh-cfg-2')) const changed = sshStore.importFromSshConfig() diff --git a/src/main/persistence.ts b/src/main/persistence.ts index 37108fbbce9..65b83de72bc 100644 --- a/src/main/persistence.ts +++ b/src/main/persistence.ts @@ -1113,6 +1113,7 @@ function backfillLegacyAutomationContexts( type LegacySshTarget = SshTarget & { remoteWorkspaceSyncEnabled?: unknown remoteWorkspaceSyncGracePeriodSeconds?: unknown + experimentalPtySourceCreditV1?: unknown } // Why: old targets predate configHost; default to label-based lookup so imported SSH aliases still resolve via ssh -G. @@ -1127,6 +1128,7 @@ function normalizeSshTarget(t: SshTarget): SshTarget { delete target.remoteWorkspaceSyncGracePeriodSeconds delete target.relayGracePeriodSeconds delete target.systemSshConnectionReuse + delete target.experimentalPtySourceCreditV1 // Why: prefer the synced grace over stale relayGracePeriodSeconds so a user's "unlimited" (0) survives migration. const relayGracePeriodSeconds = legacySyncEnabled === true && typeof legacyGracePeriodSeconds === 'number' diff --git a/src/main/providers/pty-spawn-result.ts b/src/main/providers/pty-spawn-result.ts index 4521925730e..6608b63a7fd 100644 --- a/src/main/providers/pty-spawn-result.ts +++ b/src/main/providers/pty-spawn-result.ts @@ -2,6 +2,7 @@ import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges import type { TuiAgent } from '../../shared/types' import type { AgentSessionClaimedSpawnResult } from '../../shared/agent-session-host-authority' import type { PtyIncarnationId } from '../../shared/pty-incarnation' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' export type PtySpawnResult = { agentSessionEnsure?: AgentSessionClaimedSpawnResult @@ -10,6 +11,8 @@ export type PtySpawnResult = { id: string /** Opaque provider-owned identity for this process behind a reusable PTY id. */ incarnationId?: PtyIncarnationId + /** Relay source identity installed before adjacent source frames are decoded. */ + sourceActivation?: PtySourceReceivingActivation /** The provider observed this exact spawn exit before its control reply settled. */ exitedBeforeSpawnReply?: true /** OS-level pid of the shell process, when available at spawn time. diff --git a/src/main/providers/ssh-agent-session-create-operation.ts b/src/main/providers/ssh-agent-session-create-operation.ts index 9b8d1d88415..adc0eb264d8 100644 --- a/src/main/providers/ssh-agent-session-create-operation.ts +++ b/src/main/providers/ssh-agent-session-create-operation.ts @@ -2,6 +2,14 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION } from '../../shared/agent-session-host-authority' import { isPtyIncarnationId } from '../../shared/pty-incarnation' import type { PtySpawnResult } from './pty-spawn-result' +import type { PtySpawnOptions } from './types' +import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' +import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing' +import { + parsePtySourceReceivingActivation, + type PtySourceReceivingActivation +} from '../../shared/pty-source-receiving-activation' +import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation' export const SSH_AGENT_SESSION_CAPABILITY_PROBE_TIMEOUT_MS = 5_000 @@ -49,11 +57,14 @@ export async function requestSshAgentSessionCreate(args: { params: Record operationId?: string signal?: AbortSignal + beforeResolve?: (result: unknown) => void }): Promise { try { - return await (args.signal - ? args.mux.request('pty.spawn', args.params, { signal: args.signal }) - : args.mux.request('pty.spawn', args.params)) + const options = + args.signal || args.beforeResolve + ? { signal: args.signal, beforeResolve: args.beforeResolve } + : undefined + return await args.mux.request('pty.spawn', args.params, options) } catch (error) { if (!args.operationId) { throw error @@ -63,3 +74,111 @@ export async function requestSshAgentSessionCreate(args: { throw Object.assign(spawnError, { agentSessionOperationOutcome: 'unknown' as const }) } } + +export async function spawnFreshSshPty(args: { + mux: SshChannelMultiplexer + options: PtySpawnOptions + params: Record + exitRaceTracker: SshPtySpawnExitRaceTracker + installSourceActivation: ( + relayPtyId: string, + activation: PtySourceReceivingActivation + ) => SshPtyReceivingActivationLease + rememberPtyIncarnation: (relayPtyId: string, incarnationId: unknown) => void + acceptLivePty: (appPtyId: string) => void + toAppPtyId: (relayPtyId: string) => string +}): Promise { + const operation = args.exitRaceTracker.begin() + let sourceActivationLease: SshPtyReceivingActivationLease | undefined + try { + const result = await requestSshAgentSessionCreate({ + mux: args.mux, + operationId: args.options.agentSessionCreateOperationId, + signal: args.options.signal, + params: args.params, + beforeResolve: (value) => { + sourceActivationLease = installSpawnSourceActivation(value, args.installSourceActivation) + } + }) + if (args.options.agentSessionCreateOperationId) { + assertSshAgentSessionCreateResult(result) + } + const spawnResult = parseSshPtySpawnResult(result) + if (args.exitRaceTracker.didMatchingExitArrive(operation, spawnResult)) { + throw Object.assign(new Error('agent_session_exited_during_start'), { + agentSessionOperationOutcome: 'unknown' as const + }) + } + const claimed = spawnResult.agentSessionEnsure + if (args.options.agentSessionEnsure) { + const validation = validateClaimedSshSpawn(spawnResult, args.options.agentSessionEnsure) + if (!validation.valid) { + if (validation.cleanup === 'created' && typeof spawnResult.id === 'string') { + try { + await args.mux.request('pty.shutdown', { id: spawnResult.id, immediate: true }) + } catch { + throw new Error('execution_owner_unavailable') + } + } + throw new Error(validation.error) + } + } + const id = args.toAppPtyId(spawnResult.id) + args.rememberPtyIncarnation(spawnResult.id, spawnResult.incarnationId) + args.acceptLivePty(id) + const mappedResult = { + ...spawnResult, + id, + ...(claimed + ? { + agentSessionEnsure: { + ...claimed, + owner: { ...claimed.owner, ptyId: args.toAppPtyId(claimed.owner.ptyId) } + } + } + : {}) + } + sourceActivationLease?.commit() + return mappedResult + } catch (error) { + if (sourceActivationLease && !(await sourceActivationLease.rollback())) { + throw new Error('execution_owner_unavailable') + } + throw error + } finally { + args.exitRaceTracker.finish(operation) + } +} + +function installSpawnSourceActivation( + value: unknown, + install: ( + relayPtyId: string, + activation: PtySourceReceivingActivation + ) => SshPtyReceivingActivationLease +): SshPtyReceivingActivationLease | undefined { + const result = parseSshPtySpawnResult(value) + const activation = result.sourceActivation + if (!activation) { + return undefined + } + return install(result.id, activation) +} + +function parseSshPtySpawnResult(value: unknown): PtySpawnResult { + const result = + typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as PtySpawnResult) + : ({} as PtySpawnResult) + const activation = parsePtySourceReceivingActivation(result.sourceActivation) + if ( + activation && + (typeof result.id !== 'string' || + result.id.length === 0 || + !isPtyIncarnationId(result.incarnationId) || + activation.ptyIncarnation !== result.incarnationId) + ) { + throw new Error('Invalid SSH PTY source activation identity') + } + return activation ? { ...result, sourceActivation: activation } : result +} diff --git a/src/main/providers/ssh-pty-notification-routing.test.ts b/src/main/providers/ssh-pty-notification-routing.test.ts index 2c1ab863295..5fb49553692 100644 --- a/src/main/providers/ssh-pty-notification-routing.test.ts +++ b/src/main/providers/ssh-pty-notification-routing.test.ts @@ -1,13 +1,16 @@ import { describe, expect, it, vi } from 'vitest' import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' type MockMux = { onNotification: ReturnType + request: ReturnType } function createSubscription() { const mux: MockMux = { - onNotification: vi.fn() + onNotification: vi.fn(), + request: vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 })) } const dataListeners = new Set<(payload: { id: string; data: string }) => void>() const replayListeners = new Set<(payload: { id: string; data: string }) => void>() @@ -15,15 +18,18 @@ function createSubscription() { const livePtyIds = new Set() const recordExit = vi.fn() const toAppPtyId = vi.fn((id: string) => `ssh:conn@@${id}`) + const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`) - subscribeSshPtyNotifications({ + const subscription = subscribeSshPtyNotifications({ mux: mux as never, toAppPtyId, dataListeners: dataListeners as never, replayListeners: replayListeners as never, exitListeners: exitListeners as never, livePtyIds, - recordExit + recordExit, + providerGeneration: 7, + resolvePtyIncarnation }) const handler = mux.onNotification.mock.calls[0]?.[0] as ( @@ -36,15 +42,33 @@ function createSubscription() { return { handler, + mux, toAppPtyId, dataListeners, replayListeners, exitListeners, livePtyIds, - recordExit + recordExit, + resolvePtyIncarnation, + installReceivingActivation: subscription.installReceivingActivation } } +function sourceActivation( + overrides: Partial = {} +): PtySourceReceivingActivation { + return Object.freeze({ + status: 'pending', + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + checkpointSourceEndSu: 0, + recoveryEndSu: 0, + ...overrides + }) +} + describe('subscribeSshPtyNotifications', () => { it('ignores non-PTY notifications without mapping params.id', () => { const { handler, toAppPtyId } = createSubscription() @@ -70,6 +94,8 @@ describe('subscribeSshPtyNotifications', () => { expect(onData).toHaveBeenCalledWith({ id: 'ssh:conn@@pty-1', data: 'hello', + providerGeneration: 7, + ptyIncarnation: 'incarnation:pty-1', sequenceChars: 5, seq: 9 }) @@ -92,10 +118,689 @@ describe('subscribeSshPtyNotifications', () => { expect(onExit).toHaveBeenCalledWith({ id: 'ssh:conn@@pty-1', code: 0, + providerGeneration: 7, + ptyIncarnation: 'incarnation:pty-1', incarnationId: 'incarnation-1' }) }) + it('derives exact immutable source ranges and cancels malformed frames without side effects', () => { + const { + handler, + mux, + dataListeners, + livePtyIds, + toAppPtyId, + resolvePtyIncarnation, + installReceivingActivation + } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + livePtyIds.add('ssh:conn@@unrelated') + installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 10, recoveryEndSu: 14 }) + ).commit() + + handler('pty.data', { + id: 'pty-1', + data: 'data', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 14, + sourceLengthSu: 4 + }) + const acceptedSource = onData.mock.calls[0]?.[0].source + expect(Object.isFrozen(acceptedSource)).toBe(true) + const liveBeforeMalformed = new Set(livePtyIds) + toAppPtyId.mockClear() + resolvePtyIncarnation.mockClear() + handler('pty.data', { + id: 'pty-1', + data: 'bad', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 17, + sourceLengthSu: 4 + }) + + expect(onData.mock.calls[0]?.[0]).toMatchObject({ + source: { + relayPtyId: 'pty-1', + spanId: 'token-1:10:14', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1', + sourceStartSu: 10, + sourceEndSu: 14 + } + }) + expect(onData).toHaveBeenCalledTimes(1) + expect(livePtyIds).toEqual(liveBeforeMalformed) + expect(toAppPtyId).not.toHaveBeenCalled() + expect(resolvePtyIncarnation).not.toHaveBeenCalled() + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + }) + + it('keeps exact source incarnation independent without mutating legacy delivery state', () => { + const { handler, dataListeners, resolvePtyIncarnation, installReceivingActivation } = + createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + handler('pty.data', { id: 'pty-1', data: 'legacy' }) + installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 0, recoveryEndSu: 4 }) + ).commit() + handler('pty.data', { + id: 'pty-1', + data: 'data', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 4, + sourceLengthSu: 4 + }) + + expect(onData.mock.calls.map(([payload]) => payload.ptyIncarnation)).toEqual([ + 'incarnation:pty-1', + 'incarnation-1' + ]) + expect(resolvePtyIncarnation).toHaveBeenCalledTimes(1) + expect(resolvePtyIncarnation).toHaveBeenCalledWith('pty-1', undefined) + }) + + it('drops stale delivery generations without touching their PTY or unrelated PTYs', () => { + const { + handler, + mux, + dataListeners, + livePtyIds, + toAppPtyId, + resolvePtyIncarnation, + installReceivingActivation + } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + livePtyIds.add('ssh:conn@@unrelated') + installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 4, + ownerGeneration: 5, + deliveryToken: 'token-new', + recoveryEndSu: 3 + }) + ).commit() + + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 4, + ownerGeneration: 5, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + const liveBeforeStale = new Set(livePtyIds) + toAppPtyId.mockClear() + resolvePtyIncarnation.mockClear() + + handler('pty.data', { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + expect(onData).toHaveBeenCalledTimes(1) + expect(livePtyIds).toEqual(liveBeforeStale) + expect(toAppPtyId).not.toHaveBeenCalled() + expect(resolvePtyIncarnation).not.toHaveBeenCalled() + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-old' + }) + }) + + it('rejects same-generation token changes and source discontinuities', () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const sourceParams = { + id: 'pty-1', + ptyIncarnation: 'incarnation-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceLengthSu: 3 + } + installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit() + + handler('pty.data', { + ...sourceParams, + data: 'one', + deliveryToken: 'token-1', + sourceEndSu: 3 + }) + handler('pty.data', { + ...sourceParams, + data: 'two', + deliveryToken: 'token-2', + sourceEndSu: 6 + }) + handler('pty.data', { + ...sourceParams, + data: 'gap', + deliveryToken: 'token-1', + sourceEndSu: 9 + }) + handler('pty.data', { + ...sourceParams, + data: 'two', + deliveryToken: 'token-1', + sourceEndSu: 6 + }) + + expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['one', 'two']) + expect(mux.request).toHaveBeenCalledTimes(2) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-2' + }) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + }) + + it('accepts a strictly newer rotation, rejects late old data, and preserves new continuity', () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const frame = ( + data: string, + deliveryToken: string, + clientGeneration: number, + ownerGeneration: number, + sourceEndSu: number + ) => ({ + id: 'pty-1', + data, + ptyIncarnation: 'incarnation-1', + deliveryToken, + clientGeneration, + ownerGeneration, + sourceEndSu, + sourceLengthSu: data.length + }) + + installReceivingActivation( + 'pty-1', + sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 }) + ).commit() + handler('pty.data', frame('old', 'token-old', 2, 3, 3)) + installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new', + checkpointSourceEndSu: 10, + recoveryEndSu: 13 + }) + ).commit() + handler('pty.data', frame('new', 'token-new', 3, 4, 13)) + handler('pty.data', frame('old', 'token-old', 2, 3, 6)) + handler('pty.data', frame('next', 'token-new', 3, 4, 17)) + + expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['old', 'new', 'next']) + expect(mux.request).toHaveBeenCalledTimes(1) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-old' + }) + }) + + it.each([ + ['client-only advance', 3, 3, 'token-client'], + ['owner-only advance', 2, 4, 'token-owner'], + ['crossed generations', 3, 2, 'token-crossed'], + ['replayed client generation', 1, 4, 'token-replayed'], + ['reused token on newer generations', 3, 4, 'token-current'] + ])( + 'rejects a %s without replacing the accepted continuity record', + (_case, clientGeneration, ownerGeneration, deliveryToken) => { + const { + handler, + mux, + dataListeners, + livePtyIds, + toAppPtyId, + resolvePtyIncarnation, + installReceivingActivation + } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const base = { + id: 'pty-1', + ptyIncarnation: 'incarnation-1', + sourceLengthSu: 3 + } + installReceivingActivation( + 'pty-1', + sourceActivation({ deliveryToken: 'token-current', recoveryEndSu: 3 }) + ).commit() + handler('pty.data', { + ...base, + data: 'one', + deliveryToken: 'token-current', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3 + }) + const liveBeforeInvalid = new Set(livePtyIds) + toAppPtyId.mockClear() + resolvePtyIncarnation.mockClear() + + handler('pty.data', { + ...base, + data: 'bad', + deliveryToken, + clientGeneration, + ownerGeneration, + sourceEndSu: 6 + }) + handler('pty.data', { + ...base, + data: 'two', + deliveryToken: 'token-current', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 6 + }) + + expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['one', 'two']) + expect(livePtyIds).toEqual(liveBeforeInvalid) + expect(toAppPtyId).toHaveBeenCalledTimes(1) + expect(resolvePtyIncarnation).not.toHaveBeenCalled() + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration, + ownerGeneration, + deliveryToken + }) + } + ) + + it('does not cancel an incomplete malformed identity or mutate provider state', () => { + const { handler, mux, dataListeners, livePtyIds, toAppPtyId, resolvePtyIncarnation } = + createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + livePtyIds.add('ssh:conn@@unrelated') + + handler('pty.data', { + id: 'pty-1', + data: 'bad', + ptyIncarnation: 'incarnation-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated'])) + expect(toAppPtyId).not.toHaveBeenCalled() + expect(resolvePtyIncarnation).not.toHaveBeenCalled() + expect(mux.request).not.toHaveBeenCalled() + }) + + it('accepts non-empty recovery from the activation checkpoint', () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) + ) + + handler('pty.data', { + id: 'pty-1', + data: 'next', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + expect(onData).not.toHaveBeenCalled() + lease.commit() + expect(onData).toHaveBeenCalledWith( + expect.objectContaining({ + data: 'next', + source: expect.objectContaining({ sourceStartSu: 4, sourceEndSu: 8 }) + }) + ) + }) + + it('routes held and later recovery frames only to the private sink until commit', () => { + const { handler, dataListeners, livePtyIds, installReceivingActivation } = createSubscription() + const onData = vi.fn() + const onRecoveryData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 12 }) + ) + const publishSource = (data: string, sourceEndSu: number): void => { + handler('pty.data', { + id: 'pty-1', + data, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu, + sourceLengthSu: 4 + }) + } + + publishSource('held', 8) + const recoveryLease = lease.transferToRecovery(onRecoveryData) + publishSource('next', 12) + + expect(onRecoveryData.mock.calls.map(([payload]) => payload.data)).toEqual(['held', 'next']) + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + + recoveryLease.commit() + expect(onData).not.toHaveBeenCalled() + publishSource('live', 16) + + expect(onRecoveryData).toHaveBeenCalledTimes(2) + expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'live' })) + expect(livePtyIds).toContain('ssh:conn@@pty-1') + }) + + it('retires an exited private recovery when its activation commits', () => { + const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = + createSubscription() + const onData = vi.fn() + const onRecoveryData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 4 })) + handler('pty.data', { + id: 'pty-1', + data: 'held', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 4, + sourceLengthSu: 4 + }) + const recoveryLease = lease.transferToRecovery(onRecoveryData) + + handler('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' }) + recoveryLease.commit() + handler('pty.data', { + id: 'pty-1', + data: 'late', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + expect(onRecoveryData).toHaveBeenCalledOnce() + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + expect(mux.request).toHaveBeenCalledWith( + 'pty.cancelDelivery', + expect.objectContaining({ id: 'pty-1', deliveryToken: 'token-1' }) + ) + }) + + it('retires private recovery locally and restores the exact predecessor', () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + const onRecoveryData = vi.fn() + dataListeners.add(onData) + installReceivingActivation( + 'pty-1', + sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 }) + ).commit() + handler('pty.data', { + id: 'pty-1', + data: 'pre', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + const replacement = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new', + checkpointSourceEndSu: 3, + recoveryEndSu: 6 + }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + replacement.transferToRecovery(onRecoveryData).retire() + handler('pty.data', { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + expect(onRecoveryData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) + expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) + expect(mux.request).not.toHaveBeenCalled() + }) + + it('rejects a stale activation without disturbing current continuity', () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit() + + expect(() => + installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 1, + ownerGeneration: 4, + deliveryToken: 'token-stale', + checkpointSourceEndSu: 3, + recoveryEndSu: 3 + }) + ) + ).toThrow('ssh_source_receiving_activation_stale') + + handler('pty.data', { + id: 'pty-1', + data: 'one', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + expect(onData).toHaveBeenCalledOnce() + }) + + it('drops provisional frames and settles cancellation before rollback completes', async () => { + const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } = + createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const lease = installReceivingActivation( + 'pty-1', + sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'next', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 8, + sourceLengthSu: 4 + }) + + await expect(lease.rollback()).resolves.toBe(true) + + expect(onData).not.toHaveBeenCalled() + expect(livePtyIds).not.toContain('ssh:conn@@pty-1') + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + }) + + it('restores the exact prior cursor when a replacement rolls back after frames', async () => { + const { handler, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + installReceivingActivation('pty-1', sourceActivation({ deliveryToken: 'token-old' })).commit() + handler('pty.data', { + id: 'pty-1', + data: 'pre', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + const replacement = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new', + checkpointSourceEndSu: 3, + recoveryEndSu: 3 + }) + ) + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + await replacement.rollback() + handler('pty.data', { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 6, + sourceLengthSu: 3 + }) + + expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old']) + }) + + it('does not let an older lease rollback replace a newer activation', async () => { + const { handler, mux, dataListeners, installReceivingActivation } = createSubscription() + const onData = vi.fn() + dataListeners.add(onData) + const older = installReceivingActivation('pty-1', sourceActivation()) + const newer = installReceivingActivation( + 'pty-1', + sourceActivation({ + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-new' + }) + ) + + await older.rollback() + handler('pty.data', { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + clientGeneration: 3, + ownerGeneration: 4, + sourceEndSu: 3, + sourceLengthSu: 3 + }) + newer.commit() + + expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' })) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-1' + }) + expect(mux.request).not.toHaveBeenCalledWith( + 'pty.cancelDelivery', + expect.objectContaining({ deliveryToken: 'token-new' }) + ) + }) + it('ignores PTY methods with missing ids', () => { const { handler, toAppPtyId, dataListeners } = createSubscription() const onData = vi.fn() @@ -105,4 +810,49 @@ describe('subscribeSshPtyNotifications', () => { expect(toAppPtyId).not.toHaveBeenCalled() expect(onData).not.toHaveBeenCalled() }) + + it('leaves recovery and cancellation control methods to their dedicated handlers', () => { + const { + handler, + mux, + toAppPtyId, + dataListeners, + replayListeners, + exitListeners, + livePtyIds, + recordExit, + resolvePtyIncarnation + } = createSubscription() + const onData = vi.fn() + const onReplay = vi.fn() + const onExit = vi.fn() + dataListeners.add(onData) + replayListeners.add(onReplay) + exitListeners.add(onExit) + livePtyIds.add('ssh:conn@@unrelated') + + for (const method of [ + 'pty.recoveryData', + 'pty.recoveryComplete', + 'pty.restoreRequired', + 'pty.deliveryCanceled' + ]) { + handler(method, { + id: 'pty-1', + data: 'control', + deliveryToken: 'token-1', + clientGeneration: 2, + ownerGeneration: 3 + }) + } + + expect(toAppPtyId).not.toHaveBeenCalled() + expect(resolvePtyIncarnation).not.toHaveBeenCalled() + expect(recordExit).not.toHaveBeenCalled() + expect(onData).not.toHaveBeenCalled() + expect(onReplay).not.toHaveBeenCalled() + expect(onExit).not.toHaveBeenCalled() + expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated'])) + expect(mux.request).not.toHaveBeenCalled() + }) }) diff --git a/src/main/providers/ssh-pty-notification-routing.ts b/src/main/providers/ssh-pty-notification-routing.ts index 1169674211f..8cd5c2cb86f 100644 --- a/src/main/providers/ssh-pty-notification-routing.ts +++ b/src/main/providers/ssh-pty-notification-routing.ts @@ -1,12 +1,35 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' import { isPtyIncarnationId } from '../../shared/pty-incarnation' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' import type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback } from './ssh-pty-provider-contract' +import { parseSshPtySourceFrame } from './ssh-pty-source-frame' +import { + SshPtySourceDeliveryLedger, + type PendingSshPtySourceData +} from './ssh-pty-source-delivery-ledger' export type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback } +export type SshPtyRecoveryActivationLease = Readonly<{ + commit: () => void + retire: () => void +}> +export type SshPtyReceivingActivationLease = Readonly<{ + commit: () => void + rollback: () => Promise + transferToRecovery: (sink: SshPtyDataCallback) => SshPtyRecoveryActivationLease +}> + +export type SshPtyNotificationSubscription = Readonly<{ + dispose: () => void + installReceivingActivation: ( + relayPtyId: string, + activation: PtySourceReceivingActivation + ) => SshPtyReceivingActivationLease +}> export function subscribeSshPtyNotifications(args: { mux: SshChannelMultiplexer @@ -16,8 +39,36 @@ export function subscribeSshPtyNotifications(args: { exitListeners: Set livePtyIds: Set recordExit: (relayPtyId: string, incarnationId: unknown) => void -}): () => void { - return args.mux.onNotification((method, params) => { + providerGeneration: number + resolvePtyIncarnation: (relayPtyId: string, incarnationId?: unknown) => string +}): SshPtyNotificationSubscription { + const toDataPayload = (pending: PendingSshPtySourceData): Parameters[0] => { + const id = args.toAppPtyId(pending.relayPtyId) + const ptyIncarnation = pending.source + ? (pending.params.ptyIncarnation as string) + : args.resolvePtyIncarnation(pending.relayPtyId, pending.params.incarnationId) + return { + id, + data: pending.data, + providerGeneration: args.providerGeneration, + ptyIncarnation, + ...(typeof pending.params.rawLength === 'number' + ? { sequenceChars: pending.params.rawLength } + : {}), + ...(pending.params.transformed === true ? { transformed: true } : {}), + ...(typeof pending.params.seq === 'number' ? { seq: pending.params.seq } : {}), + ...(pending.source ? { source: pending.source } : {}) + } + } + const publishData = (pending: PendingSshPtySourceData): void => { + const payload = toDataPayload(pending) + args.livePtyIds.add(payload.id) + for (const listener of args.dataListeners) { + listener(payload) + } + } + const sourceDeliveries = new SshPtySourceDeliveryLedger(args.mux, publishData) + const dispose = args.mux.onNotification((method, params) => { // Why: mux delivers every method to generic handlers; non-PTY payloads // (workspace.changed, fs.changed, …) have no `id` and must not reach // toAppPtyId → startsWith. @@ -28,14 +79,18 @@ export function subscribeSshPtyNotifications(args: { return } const relayPtyId = params.id - const id = args.toAppPtyId(relayPtyId) if (method === 'pty.exit') { + const id = args.toAppPtyId(relayPtyId) + const ptyIncarnation = args.resolvePtyIncarnation(relayPtyId, params.incarnationId) args.recordExit(relayPtyId, params.incarnationId) args.livePtyIds.delete(id) + sourceDeliveries.recordExit(relayPtyId) for (const listener of args.exitListeners) { listener({ id, code: params.code as number, + providerGeneration: args.providerGeneration, + ptyIncarnation, ...(isPtyIncarnationId(params.incarnationId) ? { incarnationId: params.incarnationId } : {}) @@ -43,21 +98,77 @@ export function subscribeSshPtyNotifications(args: { } return } - args.livePtyIds.add(id) if (method === 'pty.replay') { + const id = args.toAppPtyId(relayPtyId) + args.livePtyIds.add(id) for (const listener of args.replayListeners) { listener({ id, data: params.data as string }) } return } - for (const listener of args.dataListeners) { - listener({ - id, - data: params.data as string, - ...(typeof params.rawLength === 'number' ? { sequenceChars: params.rawLength } : {}), - ...(params.transformed === true ? { transformed: true } : {}), - ...(typeof params.seq === 'number' ? { seq: params.seq } : {}) + const data = typeof params.data === 'string' ? params.data : '' + const sourceFrame = parseSshPtySourceFrame(params, data, relayPtyId) + if (sourceFrame.malformed) { + cancelExactSourceDelivery(args.mux, relayPtyId, params) + return + } + const pending = Object.freeze({ + relayPtyId, + params, + data, + source: sourceFrame.source + }) + if (sourceFrame.source) { + if (!sourceDeliveries.admit({ ...pending, source: sourceFrame.source })) { + cancelExactSourceDelivery(args.mux, relayPtyId, params) + } + return + } + publishData(pending) + }) + return Object.freeze({ + dispose, + installReceivingActivation: (relayPtyId, activation) => { + const lease = sourceDeliveries.install(relayPtyId, activation) + return Object.freeze({ + commit: lease.commit, + rollback: lease.rollback, + transferToRecovery: (sink: SshPtyDataCallback) => + lease.transferToRecovery((pending) => sink(toDataPayload(pending))) }) } }) } + +function cancelExactSourceDelivery( + mux: SshChannelMultiplexer, + relayPtyId: string, + params: { + deliveryToken?: unknown + clientGeneration?: unknown + ownerGeneration?: unknown + } +): void { + if ( + typeof params.deliveryToken !== 'string' || + params.deliveryToken.length === 0 || + !positiveSafeInteger(params.clientGeneration) || + !positiveSafeInteger(params.ownerGeneration) + ) { + return + } + try { + void mux + .request('pty.cancelDelivery', { + id: relayPtyId, + clientGeneration: params.clientGeneration, + ownerGeneration: params.ownerGeneration, + deliveryToken: params.deliveryToken + }) + .catch(() => {}) + } catch {} +} + +function positiveSafeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) > 0 +} diff --git a/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts b/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts index 25d456d9407..e088b21f025 100644 --- a/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts +++ b/src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts @@ -1,7 +1,76 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION } from '../../shared/agent-session-host-authority' +import { + AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION, + AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION +} from '../../shared/agent-session-host-authority' +import { SshChannelMultiplexer, type MultiplexerTransport } from '../ssh/ssh-channel-multiplexer' +import { encodeFrame, HEADER_LENGTH, MessageType } from '../ssh/relay-protocol' import { SshPtyProvider } from './ssh-pty-provider' +function createTransport(): MultiplexerTransport & { + deliver: (data: Buffer) => void + written: Buffer[] +} { + let deliver = (_data: Buffer): void => {} + const written: Buffer[] = [] + return { + write: (data) => { + written.push(data) + }, + onData: (callback) => { + deliver = callback + }, + onClose: () => {}, + deliver: (data) => deliver(data), + written + } +} + +function rpcFrame(payload: Record, sequence: number): Buffer { + return encodeFrame(MessageType.Regular, sequence, 0, Buffer.from(JSON.stringify(payload))) +} + +function responseFrame(id: number, result: unknown, sequence: number): Buffer { + return rpcFrame({ jsonrpc: '2.0', id, result }, sequence) +} + +function notificationFrame( + method: string, + params: Record, + sequence: number +): Buffer { + return rpcFrame({ jsonrpc: '2.0', method, params }, sequence) +} + +function requestPayloads(transport: ReturnType): Record[] { + return transport.written.flatMap((frame) => { + if (frame[0] !== MessageType.Regular) { + return [] + } + const payloadLength = frame.readUInt32BE(9) + return [ + JSON.parse(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + payloadLength).toString()) as Record< + string, + unknown + > + ] + }) +} + +async function waitForRequest( + transport: ReturnType, + method: string +): Promise> { + for (let turn = 0; turn < 10; turn += 1) { + const request = requestPayloads(transport).find((payload) => payload.method === method) + if (request) { + return request + } + await Promise.resolve() + } + throw new Error(`request not dispatched: ${method}`) +} + describe('SSH fresh agent-session create operations', () => { const request = vi.fn() let provider: SshPtyProvider @@ -35,14 +104,19 @@ describe('SSH fresh agent-session create operations', () => { signal: undefined, timeoutMs: 5_000 }) - expect(request).toHaveBeenNthCalledWith(2, 'pty.spawn', { - cols: 80, - rows: 24, - cwd: undefined, - env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' }, - command: 'codex', - agentSessionCreateOperationId: 'a'.repeat(43) - }) + expect(request).toHaveBeenNthCalledWith( + 2, + 'pty.spawn', + { + cols: 80, + rows: 24, + cwd: undefined, + env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' }, + command: 'codex', + agentSessionCreateOperationId: 'a'.repeat(43) + }, + expect.objectContaining({ beforeResolve: expect.any(Function) }) + ) }) it('does not downgrade after structured dispatch reaches an old relay', async () => { @@ -70,13 +144,18 @@ describe('SSH fresh agent-session create operations', () => { }) ).resolves.toMatchObject({ id: 'ssh:conn-1@@pty-legacy' }) - expect(request).toHaveBeenNthCalledWith(1, 'pty.spawn', { - cols: 80, - rows: 24, - cwd: undefined, - env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' }, - command: 'codex' - }) + expect(request).toHaveBeenNthCalledWith( + 1, + 'pty.spawn', + { + cols: 80, + rows: 24, + cwd: undefined, + env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' }, + command: 'codex' + }, + expect.objectContaining({ beforeResolve: expect.any(Function) }) + ) }) it('re-probes a negative capability after an in-place relay upgrade', async () => { @@ -155,4 +234,163 @@ describe('SSH fresh agent-session create operations', () => { }) expect(request).toHaveBeenCalledTimes(2) }) + + it('withholds same-turn source data until claim validation and isolates rollback', async () => { + const transport = createTransport() + const mux = new SshChannelMultiplexer(transport) + const exactProvider = new SshPtyProvider('conn-1', mux) + const onData = vi.fn() + exactProvider.onData(onData) + const claim = { + digestVersion: 1 as const, + keyId: 'key', + identityDigest: 'a'.repeat(43), + worktreeScopeDigest: 'b'.repeat(43), + agent: 'codex' as const + } + const surface = { + worktreeId: 'worktree', + tabId: 'tab', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_claimed' + } + const spawn = exactProvider.spawn({ + cols: 80, + rows: 24, + agentSessionEnsure: { claim, surface } + }) + const capabilityRequest = await waitForRequest(transport, 'pty.getCapabilities') + transport.deliver( + responseFrame( + capabilityRequest.id as number, + { agentSessionClaimVersion: AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION }, + 1 + ) + ) + const spawnRequest = await waitForRequest(transport, 'pty.spawn') + const oldActivation = { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-old', + deliveryToken: 'token-old', + checkpointSourceEndSu: 0, + recoveryEndSu: 3 + } + transport.deliver( + Buffer.concat([ + responseFrame( + spawnRequest.id as number, + { + id: 'pty-1', + incarnationId: 'incarnation-old', + sourceActivation: oldActivation, + agentSessionEnsure: { + disposition: 'created', + owner: { + claim: { ...claim, identityDigest: 'c'.repeat(43) }, + generation: 'generation-old', + phase: 'live', + ptyId: 'pty-1', + surface + } + } + }, + 2 + ), + notificationFrame( + 'pty.data', + { + id: 'pty-1', + data: 'old', + ptyIncarnation: 'incarnation-old', + deliveryToken: 'token-old', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 3, + sourceLengthSu: 3 + }, + 3 + ) + ]) + ) + + expect(onData).not.toHaveBeenCalled() + expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(false) + const shutdownRequest = await waitForRequest(transport, 'pty.shutdown') + transport.deliver(responseFrame(shutdownRequest.id as number, null, 4)) + const cancelRequest = await waitForRequest(transport, 'pty.cancelDelivery') + let spawnSettled = false + void spawn.then( + () => { + spawnSettled = true + }, + () => { + spawnSettled = true + } + ) + await Promise.resolve() + expect(spawnSettled).toBe(false) + expect(cancelRequest.params).toEqual({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-old' + }) + + const replacement = exactProvider.spawn({ cols: 80, rows: 24 }) + const replacementRequest = requestPayloads(transport).findLast( + (payload) => payload.method === 'pty.spawn' + ) + expect(replacementRequest).toBeDefined() + transport.deliver( + Buffer.concat([ + responseFrame( + replacementRequest!.id as number, + { + id: 'pty-1', + incarnationId: 'incarnation-new', + sourceActivation: { + ...oldActivation, + clientGeneration: 4, + ownerGeneration: 5, + ptyIncarnation: 'incarnation-new', + deliveryToken: 'token-new' + } + }, + 5 + ), + notificationFrame( + 'pty.data', + { + id: 'pty-1', + data: 'new', + ptyIncarnation: 'incarnation-new', + deliveryToken: 'token-new', + clientGeneration: 4, + ownerGeneration: 5, + sourceEndSu: 3, + sourceLengthSu: 3 + }, + 6 + ) + ]) + ) + await expect(replacement).resolves.toMatchObject({ incarnationId: 'incarnation-new' }) + expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['new']) + + transport.deliver( + responseFrame( + cancelRequest.id as number, + { canceled: true, sentEndSu: 3, creditedEndSu: 0 }, + 7 + ) + ) + await expect(spawn).rejects.toThrow('agent_session_ownership_unknown') + expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(true) + expect( + requestPayloads(transport).filter((payload) => payload.method === 'pty.cancelDelivery') + ).toHaveLength(1) + mux.dispose() + }) }) diff --git a/src/main/providers/ssh-pty-provider-contract.ts b/src/main/providers/ssh-pty-provider-contract.ts index 9043133b03c..18c21bdb754 100644 --- a/src/main/providers/ssh-pty-provider-contract.ts +++ b/src/main/providers/ssh-pty-provider-contract.ts @@ -5,19 +5,40 @@ export type RemoteCliBridgeEnv = { relayDir: string nodePath: string sockPath: string + credentialFile?: string pathDelimiter?: ':' | ';' } export type SshPtyDataCallback = (payload: { id: string data: string + providerGeneration: number + ptyIncarnation: string sequenceChars?: number transformed?: boolean seq?: number + source?: Readonly<{ + relayPtyId: string + spanId: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string + sourceStartSu: number + sourceEndSu: number + }> + sourceMalformed?: boolean }) => void export type SshPtyReplayCallback = (payload: { id: string; data: string }) => void export type SshPtyExitCallback = (payload: { id: string code: number + providerGeneration: number + ptyIncarnation: string incarnationId?: PtyIncarnationId }) => void + +export type SshPtyDeliveryPauseAdapter = (args: { + id: string + providerGeneration: number + paused: boolean +}) => void diff --git a/src/main/providers/ssh-pty-provider-exit-race.test.ts b/src/main/providers/ssh-pty-provider-exit-race.test.ts index 12616e4d133..22a19985d21 100644 --- a/src/main/providers/ssh-pty-provider-exit-race.test.ts +++ b/src/main/providers/ssh-pty-provider-exit-race.test.ts @@ -1,6 +1,18 @@ import { expect, it, vi } from 'vitest' import { SshPtyProvider } from './ssh-pty-provider' +function sourceActivation(ptyIncarnation: string) { + return { + status: 'pending' as const, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation, + deliveryToken: `token:${ptyIncarnation}`, + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + } +} + it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async () => { const mux = { request: vi.fn(), @@ -10,17 +22,38 @@ it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async ( isDisposed: vi.fn().mockReturnValue(false) } const provider = new SshPtyProvider('conn-1', mux as never) + const dataListener = vi.fn() + provider.onData(dataListener) const exitListener = vi.fn() provider.onExit(exitListener) - mux.request.mockImplementation(async (method: string) => { + mux.request.mockImplementation(async (method: string, _params, options) => { if (method === 'pty.spawn') { + const result = { + id: 'pty-raced', + incarnationId: 'incarnation-raced', + sourceActivation: sourceActivation('incarnation-raced') + } + options?.beforeResolve?.(result) const notify = mux.onNotification.mock.calls[0]?.[0] + notify?.('pty.data', { + id: 'pty-raced', + data: 'data', + ptyIncarnation: 'incarnation-raced', + deliveryToken: 'token:incarnation-raced', + clientGeneration: 2, + ownerGeneration: 3, + sourceEndSu: 4, + sourceLengthSu: 4 + }) notify?.('pty.exit', { id: 'pty-raced', code: 0, incarnationId: 'incarnation-raced' }) - return { id: 'pty-raced', incarnationId: 'incarnation-raced' } + return result + } + if (method === 'pty.cancelDelivery') { + return { canceled: true, sentEndSu: 4, creditedEndSu: 0 } } return undefined }) @@ -32,7 +65,16 @@ it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async ( expect(exitListener).toHaveBeenCalledWith({ id: 'ssh:conn-1@@pty-raced', code: 0, - incarnationId: 'incarnation-raced' + incarnationId: 'incarnation-raced', + providerGeneration: expect.any(Number), + ptyIncarnation: 'incarnation-raced' + }) + expect(dataListener).not.toHaveBeenCalled() + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-raced', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token:incarnation-raced' }) mux.request.mockResolvedValue({ id: 'pty-next', incarnationId: 'incarnation-next' }) await expect(provider.spawn({ cols: 80, rows: 24 })).resolves.toMatchObject({ @@ -50,15 +92,20 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch', isDisposed: vi.fn().mockReturnValue(false) } const provider = new SshPtyProvider('conn-1', mux as never) - mux.request.mockImplementation(async (method: string) => { + mux.request.mockImplementation(async (method: string, _params, options) => { if (method === 'pty.attach') { + const result = { + incarnationId: 'incarnation-existing', + sourceActivation: sourceActivation('incarnation-existing') + } + options?.beforeResolve?.(result) const notify = mux.onNotification.mock.calls[0]?.[0] notify?.('pty.exit', { id: 'pty-existing', code: 0, incarnationId: 'incarnation-existing' }) - return { incarnationId: 'incarnation-existing' } + return result } return undefined }) @@ -66,6 +113,12 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch', await expect( provider.spawn({ cols: 80, rows: 24, sessionId: 'ssh:conn-1@@pty-existing' }) ).rejects.toThrow('agent_session_exited_during_start') + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-existing', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token:incarnation-existing' + }) mux.request.mockResolvedValue({ incarnationId: 'incarnation-next' }) await expect( @@ -76,3 +129,44 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch', isReattach: true }) }) + +it('returns a provisional source activation lease to reconnect authority', async () => { + const mux = { + request: vi.fn(), + notify: vi.fn(), + onNotification: vi.fn(), + dispose: vi.fn(), + isDisposed: vi.fn().mockReturnValue(false) + } + const provider = new SshPtyProvider('conn-1', mux as never) + const response = { + incarnationId: 'incarnation-reconnect', + sourceActivation: { + ...sourceActivation('incarnation-reconnect'), + deliveryToken: 'token-reconnect', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + } + } + mux.request.mockImplementation(async (method: string, _params, options) => { + if (method === 'pty.cancelDelivery') { + return { canceled: true, sentEndSu: 8, creditedEndSu: 4 } + } + if (method !== 'pty.attach') { + return undefined + } + options?.beforeResolve?.(response) + return response + }) + + const result = await provider.attachForReconnect('ssh:conn-1@@pty-1') + await result.sourceActivationLease?.rollback() + + expect(Object.isFrozen(result.sourceActivation)).toBe(true) + expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-reconnect' + }) +}) diff --git a/src/main/providers/ssh-pty-provider-output-state.ts b/src/main/providers/ssh-pty-provider-output-state.ts new file mode 100644 index 00000000000..fd9bde5255f --- /dev/null +++ b/src/main/providers/ssh-pty-provider-output-state.ts @@ -0,0 +1,147 @@ +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import type { + SshPtyDataCallback, + SshPtyDeliveryPauseAdapter, + SshPtyExitCallback, + SshPtyReplayCallback +} from './ssh-pty-provider-contract' +import { + subscribeSshPtyNotifications, + type SshPtyNotificationSubscription, + type SshPtyReceivingActivationLease +} from './ssh-pty-notification-routing' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' + +export class SshPtyProviderOutputState { + private readonly dataListeners = new Set() + private readonly replayListeners = new Set() + private readonly exitListeners = new Set() + private readonly incarnationByRelayPtyId = new Map() + private readonly pausedRelayPtyIds = new Set() + private deliveryPauseAdapter: SshPtyDeliveryPauseAdapter | null = null + private legacyIncarnationSerial = 1 + private subscription: SshPtyNotificationSubscription | null + + constructor( + private readonly providerGeneration: number, + args: { + mux: SshChannelMultiplexer + toAppPtyId: (id: string) => string + livePtyIds: Set + recordExit: (relayPtyId: string, incarnationId: unknown) => void + } + ) { + this.subscription = subscribeSshPtyNotifications({ + ...args, + dataListeners: this.dataListeners, + replayListeners: this.replayListeners, + exitListeners: this.exitListeners, + providerGeneration, + resolvePtyIncarnation: (relayPtyId, incarnationId) => + this.resolvePtyIncarnation(relayPtyId, incarnationId), + recordExit: (relayPtyId, incarnationId) => { + args.recordExit(relayPtyId, incarnationId) + this.incarnationByRelayPtyId.delete(relayPtyId) + this.pausedRelayPtyIds.delete(relayPtyId) + } + }) + } + + dispose(): void { + this.resumePausedDeliveries() + this.subscription?.dispose() + this.subscription = null + this.dataListeners.clear() + this.replayListeners.clear() + this.exitListeners.clear() + this.incarnationByRelayPtyId.clear() + this.deliveryPauseAdapter = null + } + + onData(callback: SshPtyDataCallback): () => void { + this.dataListeners.add(callback) + return () => this.dataListeners.delete(callback) + } + + onReplay(callback: SshPtyReplayCallback): () => void { + this.replayListeners.add(callback) + return () => this.replayListeners.delete(callback) + } + + onExit(callback: SshPtyExitCallback): () => void { + this.exitListeners.add(callback) + return () => this.exitListeners.delete(callback) + } + + setDeliveryPauseAdapter(adapter: SshPtyDeliveryPauseAdapter | null): void { + if (adapter !== this.deliveryPauseAdapter) { + this.resumePausedDeliveries() + } + this.deliveryPauseAdapter = adapter + } + + hasDeliveryPauseAdapter(): boolean { + return this.deliveryPauseAdapter !== null + } + + pause(id: string): void { + if (!this.deliveryPauseAdapter || this.pausedRelayPtyIds.has(id)) { + return + } + this.pausedRelayPtyIds.add(id) + this.deliveryPauseAdapter({ id, providerGeneration: this.providerGeneration, paused: true }) + } + + resume(id: string): void { + if (!this.deliveryPauseAdapter || !this.pausedRelayPtyIds.delete(id)) { + return + } + this.deliveryPauseAdapter({ id, providerGeneration: this.providerGeneration, paused: false }) + } + + installReceivingActivation( + relayPtyId: string, + activation: PtySourceReceivingActivation + ): SshPtyReceivingActivationLease { + if (!this.subscription) { + throw new Error('ssh_source_receiving_activation_disposed') + } + return this.subscription.installReceivingActivation(relayPtyId, activation) + } + + rememberPtyIncarnation(relayPtyId: string, incarnationId: unknown): void { + if ( + !this.incarnationByRelayPtyId.has(relayPtyId) && + typeof incarnationId === 'string' && + incarnationId.length > 0 + ) { + this.incarnationByRelayPtyId.set(relayPtyId, incarnationId) + } + } + + private resolvePtyIncarnation(relayPtyId: string, incarnationId: unknown): string { + this.rememberPtyIncarnation(relayPtyId, incarnationId) + let resolved = this.incarnationByRelayPtyId.get(relayPtyId) + if (!resolved) { + resolved = `legacy:${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}` + this.incarnationByRelayPtyId.set(relayPtyId, resolved) + } + return resolved + } + + private resumePausedDeliveries(): void { + const adapter = this.deliveryPauseAdapter + if (!adapter) { + this.pausedRelayPtyIds.clear() + return + } + for (const id of this.pausedRelayPtyIds) { + try { + adapter({ id, providerGeneration: this.providerGeneration, paused: false }) + } catch { + /* Generation close is the fallback cleanup proof. */ + } + } + this.pausedRelayPtyIds.clear() + } +} diff --git a/src/main/providers/ssh-pty-provider-process-events.test.ts b/src/main/providers/ssh-pty-provider-process-events.test.ts index 2d033aa40d3..e152ef47fae 100644 --- a/src/main/providers/ssh-pty-provider-process-events.test.ts +++ b/src/main/providers/ssh-pty-provider-process-events.test.ts @@ -13,7 +13,7 @@ function createMockMux(): MockMultiplexer { return { request: vi.fn().mockResolvedValue(undefined), notify: vi.fn(), - onNotification: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()), dispose: vi.fn(), isDisposed: vi.fn().mockReturnValue(false) } @@ -135,10 +135,17 @@ describe('SshPtyProvider process listings and events', () => { notify('pty.replay', { id: 'pty-1', data: 'buffered output' }) notify('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' }) - expect(dataHandler).toHaveBeenNthCalledWith(1, { id: scopedPty1, data: 'output' }) + expect(dataHandler).toHaveBeenNthCalledWith(1, { + id: scopedPty1, + data: 'output', + providerGeneration: 1, + ptyIncarnation: 'legacy:1:1:pty-1' + }) expect(dataHandler).toHaveBeenNthCalledWith(2, { id: scopedPty1, data: '', + providerGeneration: 1, + ptyIncarnation: 'legacy:1:1:pty-1', sequenceChars: 9, seq: 9, transformed: true @@ -147,10 +154,28 @@ describe('SshPtyProvider process listings and events', () => { expect(exitHandler).toHaveBeenCalledWith({ id: scopedPty1, code: 0, + providerGeneration: 1, + ptyIncarnation: 'legacy:1:1:pty-1', incarnationId: 'incarnation-1' }) }) + it('keeps fallback admission identity stable when spawn metadata arrives later', async () => { + const dataHandler = vi.fn() + provider.onData(dataHandler) + const notify = mux.onNotification.mock.calls[0][0] + + notify('pty.data', { id: 'pty-1', data: 'before-response' }) + mux.request.mockResolvedValue({ id: 'pty-1', incarnationId: 'incarnation-1' }) + await provider.spawn({ cols: 80, rows: 24 }) + notify('pty.data', { id: 'pty-1', data: 'after-response' }) + + expect(dataHandler.mock.calls.map(([payload]) => payload.ptyIncarnation)).toEqual([ + 'legacy:1:1:pty-1', + 'legacy:1:1:pty-1' + ]) + }) + it('supports listener removal, fanout, and connection namespaces', () => { const removed = vi.fn() const first = vi.fn() @@ -170,6 +195,29 @@ describe('SshPtyProvider process listings and events', () => { const other = vi.fn() otherProvider.onData(other) otherMux.onNotification.mock.calls[0][0]('pty.data', { id: 'pty-1', data: 'second' }) - expect(other).toHaveBeenCalledWith({ id: 'ssh:conn-2@@pty-1', data: 'second' }) + expect(other).toHaveBeenCalledWith({ + id: 'ssh:conn-2@@pty-1', + data: 'second', + providerGeneration: 1, + ptyIncarnation: 'legacy:1:1:pty-1' + }) + }) + + it('scopes pause delivery adapters and resumes them during cleanup', () => { + const adapter = vi.fn() + provider.setPtyDeliveryPauseAdapter(adapter) + + provider.pauseProducer(scopedPty1) + provider.pauseProducer(scopedPty1) + provider.resumeProducer(scopedPty1) + provider.pauseProducer(scopedPty1) + provider.dispose() + + expect(adapter.mock.calls).toEqual([ + [{ id: 'pty-1', providerGeneration: 1, paused: true }], + [{ id: 'pty-1', providerGeneration: 1, paused: false }], + [{ id: 'pty-1', providerGeneration: 1, paused: true }], + [{ id: 'pty-1', providerGeneration: 1, paused: false }] + ]) }) }) diff --git a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts new file mode 100644 index 00000000000..74989dc34e5 --- /dev/null +++ b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest' +import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' +import { SshPtyProvider } from './ssh-pty-provider' + +describe('SSH PTY provider session reattach incarnation', () => { + it('remembers the authoritative incarnation before a legacy exit arrives', async () => { + let notify: ((method: string, params: Record) => void) | undefined + const mux = { + request: vi.fn().mockResolvedValue({ incarnationId: 'incarnation-reattached' }), + notify: vi.fn(), + onNotification: vi.fn( + (callback: (method: string, params: Record) => void) => { + notify = callback + return vi.fn() + } + ) + } + const provider = new SshPtyProvider('conn-1', mux as never) + const onExit = vi.fn() + provider.onExit(onExit) + + await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) + notify?.('pty.exit', { id: 'pty-old', code: 0 }) + + expect(onExit).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'ssh:conn-1@@pty-old', + ptyIncarnation: 'incarnation-reattached' + }) + ) + }) + + it('fails closed when generic reattach requires source restoration', async () => { + const mux = { + request: vi.fn().mockResolvedValue({ + incarnationId: 'incarnation-reattached', + sourceRecovery: { + status: 'restoreRequired', + reason: 'checkpointUnavailable' + } + }), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( + `${SSH_SESSION_EXPIRED_ERROR}: pty-old` + ) + }) +}) diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index 99bb651ed99..387ec688afc 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -22,6 +22,14 @@ function createMockMux(): MockMultiplexer { } } +const sourceActivationRequestOptions = expect.objectContaining({ + beforeResolve: expect.any(Function) +}) + +function expectRequest(request: ReturnType, ...expected: unknown[]): void { + expect(request.mock.calls.map((call) => call.slice(0, expected.length))).toContainEqual(expected) +} + describe('SshPtyProvider', () => { let mux: MockMultiplexer let provider: SshPtyProvider @@ -184,7 +192,7 @@ describe('SshPtyProvider', () => { await expect( provider.spawn({ cols: 80, rows: 24, agentSessionEnsure: { claim, surface } }) ).rejects.toThrow('agent_session_ownership_unknown') - expect(mux.request).toHaveBeenCalledWith('pty.shutdown', { + expectRequest(mux.request, 'pty.shutdown', { id: 'pty-malformed', immediate: true }) @@ -261,7 +269,7 @@ describe('SshPtyProvider', () => { const result = await provider.spawn({ cols: 80, rows: 24 }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 80, rows: 24, cwd: undefined, @@ -289,7 +297,8 @@ describe('SshPtyProvider', () => { await provider.spawn({ cols: 80, rows: 24, startupIngress }) - expect(mux.request).toHaveBeenCalledWith( + expectRequest( + mux.request, 'pty.spawn', expect.objectContaining({ startupIngressVersion: PTY_STARTUP_INGRESS_VERSION, @@ -308,7 +317,7 @@ describe('SshPtyProvider', () => { env: { FOO: 'bar' } }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: '/home/user', @@ -326,7 +335,8 @@ describe('SshPtyProvider', () => { launchAgent: 'claude' }) - expect(mux.request).toHaveBeenCalledWith( + expectRequest( + mux.request, 'pty.spawn', expect.objectContaining({ command: 'cd /repo && custom-agent-wrapper', @@ -345,7 +355,7 @@ describe('SshPtyProvider', () => { tabId: 'tab-a' }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -365,7 +375,7 @@ describe('SshPtyProvider', () => { terminalWindowsWslDistro: 'Ubuntu' }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -384,7 +394,7 @@ describe('SshPtyProvider', () => { env: { [POWERLEVEL10K_WIZARD_DISABLE_ENV]: 'already-set' } }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -402,7 +412,7 @@ describe('SshPtyProvider', () => { envToDelete: [POWERLEVEL10K_WIZARD_DISABLE_ENV] }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -426,7 +436,7 @@ describe('SshPtyProvider', () => { envToDelete }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -452,7 +462,7 @@ describe('SshPtyProvider', () => { startupCommandDelivery: 'shell-ready' }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -478,7 +488,7 @@ describe('SshPtyProvider', () => { env: { PATH: '/usr/bin', ORCA_TERMINAL_HANDLE: 'term_ssh' } }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -509,7 +519,7 @@ describe('SshPtyProvider', () => { env: { ORCA_TERMINAL_HANDLE: 'term_ssh' } }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -540,7 +550,7 @@ describe('SshPtyProvider', () => { env: { Path: 'C:/Windows/System32;C:/Tools' } }) - expect(mux.request).toHaveBeenCalledWith('pty.spawn', { + expectRequest(mux.request, 'pty.spawn', { cols: 120, rows: 40, cwd: undefined, @@ -563,12 +573,17 @@ describe('SshPtyProvider', () => { const result = await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { - id: 'pty-old', - cols: 80, - rows: 24, - suppressReplayNotification: true - }) + expectRequest( + mux.request, + 'pty.attach', + { + id: 'pty-old', + cols: 80, + rows: 24, + suppressReplayNotification: true + }, + sourceActivationRequestOptions + ) expect(result).toEqual({ id: 'ssh:conn-1@@pty-old', isReattach: true, @@ -590,7 +605,8 @@ describe('SshPtyProvider', () => { } }) - expect(mux.request).toHaveBeenCalledWith( + expectRequest( + mux.request, 'pty.attach', expect.not.objectContaining({ startupIngress: expect.anything() }) ) @@ -606,7 +622,7 @@ describe('SshPtyProvider', () => { sessionId: 'ssh:conn-1@@pty-old' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { + expectRequest(mux.request, 'pty.attach', { id: 'pty-old', cols: 80, rows: 24, @@ -630,7 +646,7 @@ describe('SshPtyProvider', () => { tabId: 'tab-a' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { + expectRequest(mux.request, 'pty.attach', { id: 'pty-old', cols: 80, rows: 24, @@ -647,12 +663,17 @@ describe('SshPtyProvider', () => { 'SSH_SESSION_EXPIRED: pty-old' ) - expect(mux.request).toHaveBeenNthCalledWith(1, 'pty.attach', { - id: 'pty-old', - cols: 80, - rows: 24, - suppressReplayNotification: true - }) + expect(mux.request).toHaveBeenNthCalledWith( + 1, + 'pty.attach', + { + id: 'pty-old', + cols: 80, + rows: 24, + suppressReplayNotification: true + }, + sourceActivationRequestOptions + ) expect(mux.request).toHaveBeenCalledTimes(1) }) @@ -669,7 +690,7 @@ describe('SshPtyProvider', () => { it('attach sends pty.attach request', async () => { await provider.attach(scopedPty1) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { id: 'pty-1' }) + expectRequest(mux.request, 'pty.attach', { id: 'pty-1' }) }) it('attachForReconnect returns replay without relay notification', async () => { @@ -684,10 +705,18 @@ describe('SshPtyProvider', () => { replay: 'restored output', incarnationId: 'incarnation-reconnect' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { - id: 'pty-1', - suppressReplayNotification: true - }) + expectRequest( + mux.request, + 'pty.attach', + { + id: 'pty-1', + suppressReplayNotification: true + }, + expect.objectContaining({ + timeoutMs: 10_000, + beforeResolve: expect.any(Function) + }) + ) }) it('keeps missing incarnation compatible with an old relay', async () => { @@ -712,12 +741,20 @@ describe('SshPtyProvider', () => { tabId: 'tab-a' }) - expect(mux.request).toHaveBeenCalledWith('pty.attach', { - id: 'pty-1', - suppressReplayNotification: true, - expectedPaneKey: 'tab-a:leaf-a', - expectedTabId: 'tab-a' - }) + expectRequest( + mux.request, + 'pty.attach', + { + id: 'pty-1', + suppressReplayNotification: true, + expectedPaneKey: 'tab-a:leaf-a', + expectedTabId: 'tab-a' + }, + expect.objectContaining({ + timeoutMs: 10_000, + beforeResolve: expect.any(Function) + }) + ) }) it('write sends pty.data notification', () => { @@ -734,7 +771,7 @@ describe('SshPtyProvider', () => { mux.request.mockResolvedValue({ cols: 120, rows: 40 }) await expect(provider.getAppliedSize(scopedPty1)).resolves.toEqual({ cols: 120, rows: 40 }) - expect(mux.request).toHaveBeenCalledWith('pty.getSize', { id: 'pty-1' }, { timeoutMs: 1_000 }) + expectRequest(mux.request, 'pty.getSize', { id: 'pty-1' }, { timeoutMs: 1_000 }) }) it('caches only an old relay method-not-found response', async () => { @@ -759,7 +796,8 @@ describe('SshPtyProvider', () => { it('shutdown sends pty.shutdown request', async () => { await provider.shutdown(scopedPty1, { immediate: true }) - expect(mux.request).toHaveBeenCalledWith( + expectRequest( + mux.request, 'pty.shutdown', { id: 'pty-1', @@ -772,7 +810,8 @@ describe('SshPtyProvider', () => { it('shutdown forwards keepHistory: true over the relay', async () => { await provider.shutdown(scopedPty1, { immediate: true, keepHistory: true }) - expect(mux.request).toHaveBeenCalledWith( + expectRequest( + mux.request, 'pty.shutdown', { id: 'pty-1', @@ -789,7 +828,8 @@ describe('SshPtyProvider', () => { vi.useFakeTimers() try { await provider.shutdown(scopedPty1, { immediate: true, deadlineMs: Date.now() + 4321 }) - expect(mux.request).toHaveBeenCalledWith( + expectRequest( + mux.request, 'pty.shutdown', { id: 'pty-1', immediate: true, keepHistory: false }, { timeoutMs: 4321 } @@ -801,19 +841,19 @@ describe('SshPtyProvider', () => { it('sendSignal sends pty.sendSignal request', async () => { await provider.sendSignal(scopedPty1, 'SIGINT') - expect(mux.request).toHaveBeenCalledWith('pty.sendSignal', { id: 'pty-1', signal: 'SIGINT' }) + expectRequest(mux.request, 'pty.sendSignal', { id: 'pty-1', signal: 'SIGINT' }) }) it('getCwd sends pty.getCwd request', async () => { mux.request.mockResolvedValue('/home/user/project') const cwd = await provider.getCwd(scopedPty1) expect(cwd).toBe('/home/user/project') - expect(mux.request).toHaveBeenCalledWith('pty.getCwd', { id: 'pty-1' }) + expectRequest(mux.request, 'pty.getCwd', { id: 'pty-1' }) }) it('clearBuffer sends pty.clearBuffer request', async () => { await provider.clearBuffer(scopedPty1) - expect(mux.request).toHaveBeenCalledWith('pty.clearBuffer', { id: 'pty-1' }) + expectRequest(mux.request, 'pty.clearBuffer', { id: 'pty-1' }) }) it('acknowledgeDataEvent sends pty.ackData notification', () => { @@ -825,14 +865,14 @@ describe('SshPtyProvider', () => { mux.request.mockResolvedValue(true) const result = await provider.hasChildProcesses(scopedPty1) expect(result).toBe(true) - expect(mux.request).toHaveBeenCalledWith('pty.hasChildProcesses', { id: 'pty-1' }) + expectRequest(mux.request, 'pty.hasChildProcesses', { id: 'pty-1' }) }) it('getForegroundProcess returns process name', async () => { mux.request.mockResolvedValue('node') const result = await provider.getForegroundProcess(scopedPty1) expect(result).toBe('node') - expect(mux.request).toHaveBeenCalledWith('pty.getForegroundProcess', { id: 'pty-1' }) + expectRequest(mux.request, 'pty.getForegroundProcess', { id: 'pty-1' }) }) it('preserves unavailable process inspection', async () => { @@ -844,7 +884,7 @@ describe('SshPtyProvider', () => { mux.request.mockResolvedValue(inspection) await expect(provider.inspectProcess(scopedPty1)).resolves.toEqual(inspection) - expect(mux.request).toHaveBeenCalledWith('pty.inspectProcess', { id: 'pty-1' }) + expectRequest(mux.request, 'pty.inspectProcess', { id: 'pty-1' }) }) it('serializes scoped app ids using raw relay ids', async () => { @@ -853,7 +893,7 @@ describe('SshPtyProvider', () => { const result = await provider.serialize([scopedPty1]) expect(result).toBe('serialized') - expect(mux.request).toHaveBeenCalledWith('pty.serialize', { ids: ['pty-1'] }) + expectRequest(mux.request, 'pty.serialize', { ids: ['pty-1'] }) }) it('rejects scoped ids owned by another SSH connection', async () => { diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 59f20bfdf0a..d0b29398f1f 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -5,25 +5,24 @@ import { createSshPtyAppliedSizeReader } from './ssh-pty-applied-size' import type { RemoteCliBridgeEnv, SshPtyDataCallback, + SshPtyDeliveryPauseAdapter, SshPtyExitCallback, SshPtyReplayCallback } from './ssh-pty-provider-contract' -import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing' -import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation' -import { - assertSshAgentSessionCreateResult, - requestSshAgentSessionCreate -} from './ssh-agent-session-create-operation' +import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state' +import { spawnFreshSshPty } from './ssh-agent-session-create-operation' import { mapSshPtyProcessList } from './ssh-agent-session-process-list' import { - parseSshPtyAttachResult, + requestSshPtyAttach, reattachSshPtySessionWithExitFence, + type PtySourceRecoveryRequest, type SshPtyAttachResult } from './ssh-pty-session-reattach' import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request' import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities' import type { PtyProcessInspection } from './pty-process-inspection' +import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' // Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch. function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined { @@ -34,58 +33,43 @@ function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: numbe export class SshPtyProvider implements IPtyProvider { private mux: SshChannelMultiplexer private connectionId: string - private dataListeners = new Set() - private replayListeners = new Set() - private exitListeners = new Set() private livePtyIds = new Set() - // Why: stale notification callbacks must not outlive a disconnected provider. - private unsubscribeNotifications: (() => void) | null = null readonly getAppliedSize: NonNullable private readonly agentSessionCapabilities: SshAgentSessionCapabilities private spawnExitRaces = new SshPtySpawnExitRaceTracker() + private readonly outputState: SshPtyProviderOutputState constructor( connectionId: string, mux: SshChannelMultiplexer, - private readonly remoteCliBridgeEnv?: RemoteCliBridgeEnv + private readonly remoteCliBridgeEnv?: RemoteCliBridgeEnv, + readonly providerGeneration = 1 ) { this.connectionId = connectionId this.mux = mux this.agentSessionCapabilities = new SshAgentSessionCapabilities(mux) this.getAppliedSize = createSshPtyAppliedSizeReader(mux, connectionId) - this.unsubscribeNotifications = subscribeSshPtyNotifications({ + this.outputState = new SshPtyProviderOutputState(providerGeneration, { mux, toAppPtyId: (id) => this.toAppPtyId(id), - dataListeners: this.dataListeners, - replayListeners: this.replayListeners, - exitListeners: this.exitListeners, livePtyIds: this.livePtyIds, - recordExit: (relayPtyId, incarnationId) => + recordExit: (relayPtyId, incarnationId) => { this.spawnExitRaces.recordExit(relayPtyId, incarnationId) + } }) } dispose(): void { - if (this.unsubscribeNotifications) { - this.unsubscribeNotifications() - this.unsubscribeNotifications = null - } - this.dataListeners.clear() - this.replayListeners.clear() - this.exitListeners.clear() + this.outputState.dispose() this.livePtyIds.clear() } getConnectionId = (): string => this.connectionId - private toRelayPtyId(id: string): string { - return toRelaySshPtyId(this.connectionId, id) - } + private toRelayPtyId = (id: string): string => toRelaySshPtyId(this.connectionId, id) - private toAppPtyId(id: string): string { - return toAppSshPtyId(this.connectionId, id) - } + private toAppPtyId = (id: string): string => toAppSshPtyId(this.connectionId, id) async spawn(opts: PtySpawnOptions): Promise { if (opts.agentSessionEnsure && opts.sessionId) { @@ -101,15 +85,36 @@ export class SshPtyProvider implements IPtyProvider { } } if (opts.sessionId) { - const result = await reattachSshPtySessionWithExitFence({ - mux: this.mux, - connectionId: this.connectionId, - sessionId: opts.sessionId, - options: opts, - exitRaceTracker: this.spawnExitRaces - }) - this.livePtyIds.add(result.id) - return result + let result: Awaited> | undefined + try { + result = await reattachSshPtySessionWithExitFence({ + mux: this.mux, + connectionId: this.connectionId, + sessionId: opts.sessionId, + options: opts, + exitRaceTracker: this.spawnExitRaces, + installSourceActivation: (relayPtyId, activation) => + this.outputState.installReceivingActivation(relayPtyId, activation), + rememberPtyIncarnation: (relayPtyId, incarnationId) => + this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId) + }) + if (result.sourceRecovery?.status === 'restoreRequired') { + throw new Error( + `${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}` + ) + } + this.livePtyIds.add(result.id) + result.sourceActivationLease?.commit() + const { + sourceActivationLease: _lease, + sourceRecovery: _sourceRecovery, + ...spawnResult + } = result + return spawnResult + } catch (error) { + result?.sourceActivationLease?.rollback() + throw error + } } const supportsCreateOperation = opts.agentSessionCreateOperationId @@ -122,65 +127,22 @@ export class SshPtyProvider implements IPtyProvider { // Why: host routing owns legacy selection; a changed relay must not downgrade after dispatch. throw new Error('execution_owner_unavailable') } - const operation = this.spawnExitRaces.begin() - try { - const result = await requestSshAgentSessionCreate({ - mux: this.mux, - operationId: opts.agentSessionCreateOperationId, - signal: opts.signal, - params: buildSshPtySpawnRequest({ - options: opts, - remoteCliBridgeEnv: this.remoteCliBridgeEnv, - supportsCreateOperation - }) - }) - if (opts.agentSessionCreateOperationId) { - assertSshAgentSessionCreateResult(result) - } - const spawnResult = result as PtySpawnResult - if (this.spawnExitRaces.didMatchingExitArrive(operation, spawnResult)) { - // Why: relay notification can share the response batch; no controller registration may follow. - throw Object.assign(new Error('agent_session_exited_during_start'), { - agentSessionOperationOutcome: 'unknown' as const - }) - } - const claimed = spawnResult.agentSessionEnsure - if (opts.agentSessionEnsure) { - const validation = validateClaimedSshSpawn(spawnResult, opts.agentSessionEnsure) - if (!validation.valid) { - if (validation.cleanup === 'created' && typeof spawnResult.id === 'string') { - try { - // Why: immediate relay shutdown resolves only after physical exit; - // a best-effort graceful request cannot prove the duplicate is gone. - await this.mux.request('pty.shutdown', { id: spawnResult.id, immediate: true }) - } catch { - throw new Error('execution_owner_unavailable') - } - } - throw new Error(validation.error) - } - } - const id = this.toAppPtyId(spawnResult.id) - this.livePtyIds.add(id) - return { - ...spawnResult, - id, - ...(claimed - ? { - agentSessionEnsure: { - ...claimed, - owner: { - ...claimed.owner, - ptyId: this.toAppPtyId(claimed.owner.ptyId) - } - } - } - : {}), - ...(opts.sessionId ? { sessionExpired: true } : {}) - } - } finally { - this.spawnExitRaces.finish(operation) - } + return await spawnFreshSshPty({ + mux: this.mux, + options: opts, + params: buildSshPtySpawnRequest({ + options: opts, + remoteCliBridgeEnv: this.remoteCliBridgeEnv, + supportsCreateOperation + }), + exitRaceTracker: this.spawnExitRaces, + installSourceActivation: (id, activation) => + this.outputState.installReceivingActivation(id, activation), + rememberPtyIncarnation: (id, incarnation) => + this.outputState.rememberPtyIncarnation(id, incarnation), + acceptLivePty: (id) => this.livePtyIds.add(id), + toAppPtyId: this.toAppPtyId + }) } async supportsAgentSessionClaims(options: { signal?: AbortSignal } = {}): Promise { @@ -198,25 +160,46 @@ export class SshPtyProvider implements IPtyProvider { } async attach(id: string): Promise { - await this.mux.request('pty.attach', { id: this.toRelayPtyId(id) }) + const relayPtyId = this.toRelayPtyId(id) + await requestSshPtyAttach({ + mux: this.mux, + relayPtyId, + params: { id: relayPtyId }, + commitSourceActivation: true, + installSourceActivation: (ptyId, activation) => + this.outputState.installReceivingActivation(ptyId, activation), + rememberPtyIncarnation: (ptyId, incarnationId) => + this.outputState.rememberPtyIncarnation(ptyId, incarnationId) + }) } async attachForReconnect( id: string, - expected?: { paneKey?: string; tabId?: string } + expected?: { paneKey?: string; tabId?: string }, + sourceRecovery?: PtySourceRecoveryRequest ): Promise { // Why: reconnect owns replay delivery so stale/duplicate attach results can // be filtered before they reach the renderer. The expected identity lets the // relay reject a cross-generation id collision instead of reattaching this // lease to a different pane's freshly spawned PTY. - return parseSshPtyAttachResult( - await this.mux.request('pty.attach', { - id: this.toRelayPtyId(id), - suppressReplayNotification: true, - ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), - ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) - }) - ) + const params = { + id: this.toRelayPtyId(id), + suppressReplayNotification: true, + ...(sourceRecovery ? { sourceRecovery } : {}), + ...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}), + ...(expected?.tabId ? { expectedTabId: expected.tabId } : {}) + } + const relayPtyId = this.toRelayPtyId(id) + return await requestSshPtyAttach({ + mux: this.mux, + relayPtyId, + params, + timeoutMs: 10_000, + installSourceActivation: (ptyId, activation) => + this.outputState.installReceivingActivation(ptyId, activation), + rememberPtyIncarnation: (ptyId, incarnationId) => + this.outputState.rememberPtyIncarnation(ptyId, incarnationId) + }) } write(id: string, data: string): void { @@ -308,6 +291,8 @@ export class SshPtyProvider implements IPtyProvider { const processes = mapSshPtyProcessList(result as PtyProcessInfo[], (id) => this.toAppPtyId(id)) for (const process of processes) { this.livePtyIds.add(process.id) + const relayPtyId = this.toRelayPtyId(process.id) + this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId) } return processes } @@ -326,18 +311,28 @@ export class SshPtyProvider implements IPtyProvider { return result as { name: string; path: string }[] } - onData(callback: SshPtyDataCallback): () => void { - this.dataListeners.add(callback) - return () => this.dataListeners.delete(callback) + onData = (callback: SshPtyDataCallback): (() => void) => this.outputState.onData(callback) + onReplay = (callback: SshPtyReplayCallback): (() => void) => this.outputState.onReplay(callback) + onExit = (callback: SshPtyExitCallback): (() => void) => this.outputState.onExit(callback) + + setPtyDeliveryPauseAdapter(adapter: SshPtyDeliveryPauseAdapter | null): void { + this.outputState.setDeliveryPauseAdapter(adapter) } - onReplay(callback: SshPtyReplayCallback): () => void { - this.replayListeners.add(callback) - return () => this.replayListeners.delete(callback) + hasPtyDeliveryPauseAdapter(): boolean { + return this.outputState.hasDeliveryPauseAdapter() } - onExit(callback: SshPtyExitCallback): () => void { - this.exitListeners.add(callback) - return () => this.exitListeners.delete(callback) + pauseProducer(id: string): void { + this.outputState.pause(this.toRelayPtyId(id)) + } + + resumeProducer(id: string): void { + this.outputState.resume(this.toRelayPtyId(id)) + } + + closeOutputIntake(reason: string): void { + this.mux.dispose('connection_lost') + console.error('[ssh-pty-provider] closed after bounded output intake failure', { reason }) } } diff --git a/src/main/providers/ssh-pty-session-reattach.ts b/src/main/providers/ssh-pty-session-reattach.ts index 5970f85da0a..34974940caf 100644 --- a/src/main/providers/ssh-pty-session-reattach.ts +++ b/src/main/providers/ssh-pty-session-reattach.ts @@ -9,10 +9,27 @@ import { import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' import type { PtySpawnOptions, PtySpawnResult } from './types' import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' +import type { + PtySourceRecoveryRequest, + PtySourceRecoveryResult +} from '../../shared/pty-source-recovery-contract' +import { + parsePtySourceReceivingActivation, + type PtySourceReceivingActivation +} from '../../shared/pty-source-receiving-activation' +import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing' export type SshPtyAttachResult = { replay?: string incarnationId?: PtyIncarnationId + sourceRecovery?: PtySourceRecoveryResult + sourceActivation?: PtySourceReceivingActivation + sourceActivationLease?: SshPtyReceivingActivationLease +} + +type SshPtyReattachResult = PtySpawnResult & { + sourceRecovery?: PtySourceRecoveryResult + sourceActivationLease?: SshPtyReceivingActivationLease } export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult { @@ -22,7 +39,12 @@ export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult { if (typeof value !== 'object' || Array.isArray(value)) { throw new Error('Invalid SSH PTY attach response') } - const result = value as { replay?: unknown; incarnationId?: unknown } + const result = value as { + replay?: unknown + incarnationId?: unknown + sourceRecovery?: unknown + sourceActivation?: unknown + } if (result.replay !== undefined && typeof result.replay !== 'string') { throw new Error('Invalid SSH PTY attach replay') } @@ -30,34 +52,156 @@ export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult { // Why: a present-but-invalid identity cannot safely fence delayed exits from a reused relay id. throw new Error('Invalid SSH PTY attach incarnation') } + const sourceRecovery = parseSourceRecoveryResult(result.sourceRecovery) + const sourceActivation = parsePtySourceReceivingActivation(result.sourceActivation) + const activation = + sourceActivation ?? (sourceRecovery?.status === 'pending' ? sourceRecovery : undefined) + if ( + activation && + (!isPtyIncarnationId(result.incarnationId) || + activation.ptyIncarnation !== result.incarnationId || + (sourceRecovery?.status === 'pending' && !sameSourceActivation(activation, sourceRecovery))) + ) { + throw new Error('Invalid SSH PTY source activation identity') + } return { ...(typeof result.replay === 'string' ? { replay: result.replay } : {}), - ...(isPtyIncarnationId(result.incarnationId) ? { incarnationId: result.incarnationId } : {}) + ...(isPtyIncarnationId(result.incarnationId) ? { incarnationId: result.incarnationId } : {}), + ...(sourceRecovery ? { sourceRecovery } : {}), + ...(activation ? { sourceActivation: activation } : {}) } } +export async function requestSshPtyAttach(args: { + mux: SshChannelMultiplexer + relayPtyId: string + params: Record + timeoutMs?: number + commitSourceActivation?: boolean + installSourceActivation?: ( + relayPtyId: string, + activation: PtySourceReceivingActivation + ) => SshPtyReceivingActivationLease + rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void +}): Promise { + let activationLease: SshPtyReceivingActivationLease | undefined + const installFromResult = (result: SshPtyAttachResult): void => { + if (!activationLease && result.sourceActivation && args.installSourceActivation) { + activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation) + } + } + try { + const rawResult = await args.mux.request('pty.attach', args.params, { + ...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }), + beforeResolve: (value) => installFromResult(parseSshPtyAttachResult(value)) + }) + const result = parseSshPtyAttachResult(rawResult) + installFromResult(result) + args.rememberPtyIncarnation?.(args.relayPtyId, result.incarnationId) + if (args.commitSourceActivation) { + activationLease?.commit() + } + return { + ...result, + ...(activationLease ? { sourceActivationLease: activationLease } : {}) + } + } catch (error) { + activationLease?.rollback() + throw error + } +} + +function parseSourceRecoveryResult(value: unknown): PtySourceRecoveryResult | undefined { + if (value === undefined) { + return undefined + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid SSH PTY source recovery response') + } + const input = value as Record + if (input.status === 'restoreRequired' && typeof input.reason === 'string') { + return Object.freeze({ status: 'restoreRequired', reason: input.reason }) + } + if ( + input.status !== 'pending' || + typeof input.deliveryToken !== 'string' || + input.deliveryToken.length === 0 || + typeof input.ptyIncarnation !== 'string' || + input.ptyIncarnation.length === 0 || + !positiveInteger(input.clientGeneration) || + !positiveInteger(input.ownerGeneration) || + !nonNegativeInteger(input.checkpointSourceEndSu) || + !nonNegativeInteger(input.recoveryEndSu) || + Number(input.recoveryEndSu) < Number(input.checkpointSourceEndSu) + ) { + throw new Error('Invalid SSH PTY source recovery response') + } + return Object.freeze({ + status: 'pending', + deliveryToken: input.deliveryToken, + ptyIncarnation: input.ptyIncarnation, + clientGeneration: Number(input.clientGeneration), + ownerGeneration: Number(input.ownerGeneration), + checkpointSourceEndSu: Number(input.checkpointSourceEndSu), + recoveryEndSu: Number(input.recoveryEndSu) + }) +} + +function positiveInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) > 0 +} + +function nonNegativeInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) >= 0 +} + +function sameSourceActivation( + left: PtySourceReceivingActivation, + right: PtySourceReceivingActivation +): boolean { + return ( + left.clientGeneration === right.clientGeneration && + left.ownerGeneration === right.ownerGeneration && + left.ptyIncarnation === right.ptyIncarnation && + left.deliveryToken === right.deliveryToken && + left.checkpointSourceEndSu === right.checkpointSourceEndSu && + left.recoveryEndSu === right.recoveryEndSu + ) +} + +export type { PtySourceRecoveryRequest } + export async function reattachSshPtySession(args: { mux: SshChannelMultiplexer connectionId: string sessionId: string options: PtySpawnOptions -}): Promise { + rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void + installSourceActivation?: ( + relayPtyId: string, + activation: PtySourceReceivingActivation + ) => SshPtyReceivingActivationLease +}): Promise { const relaySessionId = toRelaySshPtyId(args.connectionId, args.sessionId) console.warn(`[ssh-pty] spawn() called with sessionId=${args.sessionId}, attempting pty.attach`) try { // Why: expected pane identity prevents a reused relay id from attaching the wrong shell. const expectedPaneKey = args.options.paneKey ?? args.options.env?.ORCA_PANE_KEY const expectedTabId = args.options.tabId ?? args.options.env?.ORCA_TAB_ID - const attachResult = parseSshPtyAttachResult( - await args.mux.request('pty.attach', { + const attachResult = await requestSshPtyAttach({ + mux: args.mux, + relayPtyId: relaySessionId, + params: { id: relaySessionId, cols: args.options.cols, rows: args.options.rows, suppressReplayNotification: true, ...(expectedPaneKey ? { expectedPaneKey } : {}), ...(expectedTabId ? { expectedTabId } : {}) - }) - ) + }, + installSourceActivation: args.installSourceActivation, + rememberPtyIncarnation: args.rememberPtyIncarnation + }) console.warn( `[ssh-pty] pty.attach succeeded for ${args.sessionId}, replay=${!!attachResult.replay}` ) @@ -65,7 +209,12 @@ export async function reattachSshPtySession(args: { id: toAppSshPtyId(args.connectionId, relaySessionId), isReattach: true, ...(attachResult.replay ? { replay: attachResult.replay } : {}), - ...(attachResult.incarnationId ? { incarnationId: attachResult.incarnationId } : {}) + ...(attachResult.incarnationId ? { incarnationId: attachResult.incarnationId } : {}), + ...(attachResult.sourceRecovery ? { sourceRecovery: attachResult.sourceRecovery } : {}), + ...(attachResult.sourceActivation ? { sourceActivation: attachResult.sourceActivation } : {}), + ...(attachResult.sourceActivationLease + ? { sourceActivationLease: attachResult.sourceActivationLease } + : {}) } } catch (error) { // Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding. @@ -84,10 +233,11 @@ export async function reattachSshPtySessionWithExitFence( args: Parameters[0] & { exitRaceTracker: SshPtySpawnExitRaceTracker } -): Promise { +): Promise { const operation = args.exitRaceTracker.begin() + let result: SshPtyReattachResult | undefined try { - const result = await reattachSshPtySession(args) + result = await reattachSshPtySession(args) const relayPtyId = toRelaySshPtyId(args.connectionId, result.id) if ( args.exitRaceTracker.didMatchingExitArrive(operation, { @@ -98,6 +248,9 @@ export async function reattachSshPtySessionWithExitFence( throw new Error('agent_session_exited_during_start') } return result + } catch (error) { + result?.sourceActivationLease?.rollback() + throw error } finally { args.exitRaceTracker.finish(operation) } diff --git a/src/main/providers/ssh-pty-source-delivery-ledger.test.ts b/src/main/providers/ssh-pty-source-delivery-ledger.test.ts new file mode 100644 index 00000000000..4040b56a0d3 --- /dev/null +++ b/src/main/providers/ssh-pty-source-delivery-ledger.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import { SshPtySourceDeliveryLedger } from './ssh-pty-source-delivery-ledger' + +describe('SshPtySourceDeliveryLedger', () => { + it('retains cancellation ownership when recovery transfer is superseded', async () => { + const request = vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 })) + const ledger = new SshPtySourceDeliveryLedger({ request } as never, vi.fn()) + const older = ledger.install( + 'pty-1', + Object.freeze({ + status: 'pending', + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-old', + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + }) + ) + ledger.install( + 'pty-1', + Object.freeze({ + status: 'pending', + clientGeneration: 3, + ownerGeneration: 4, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-new', + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + }) + ) + + expect(() => older.transferToRecovery(vi.fn())).toThrow('ssh_source_receiving_activation_stale') + await expect(older.rollback()).resolves.toBe(true) + + expect(request).toHaveBeenCalledOnce() + expect(request).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: 'token-old' + }) + }) +}) diff --git a/src/main/providers/ssh-pty-source-delivery-ledger.ts b/src/main/providers/ssh-pty-source-delivery-ledger.ts new file mode 100644 index 00000000000..9e101345c49 --- /dev/null +++ b/src/main/providers/ssh-pty-source-delivery-ledger.ts @@ -0,0 +1,320 @@ +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' +import type { SshPtySourceFrame } from './ssh-pty-source-frame' + +export type PendingSshPtySourceData = Readonly<{ + relayPtyId: string + params: Record + data: string + source?: SshPtySourceFrame +}> + +type SourceDeliveryLeaseState = { + phase: 'provisional' | 'recovery' | 'committing' | 'committed' | 'retired' + pendingData: PendingSshPtySourceData[] + recoverySink?: (pending: PendingSshPtySourceData) => void + exited: boolean +} + +type SourceDeliveryState = Readonly<{ + activation: PtySourceReceivingActivation + sourceEndSu: number + lease: SourceDeliveryLeaseState + previous?: SourceDeliveryState +}> + +export type SshPtyRecoveryActivationLease = Readonly<{ + commit: () => void + retire: () => void +}> + +export type SshPtySourceDeliveryLease = Readonly<{ + commit: () => void + rollback: () => Promise + transferToRecovery: ( + sink: (pending: PendingSshPtySourceData) => void + ) => SshPtyRecoveryActivationLease +}> + +export class SshPtySourceDeliveryLedger { + private readonly deliveryByPty = new Map() + + constructor( + private readonly mux: SshChannelMultiplexer, + private readonly publishData: (pending: PendingSshPtySourceData) => void + ) {} + + install(relayPtyId: string, activation: PtySourceReceivingActivation): SshPtySourceDeliveryLease { + if (!relayPtyId || activation.ptyIncarnation.length === 0) { + throw new Error('ssh_source_receiving_activation_invalid') + } + const previous = this.deliveryByPty.get(relayPtyId) + if (previous && sameReceivingActivation(previous.activation, activation)) { + if (previous.lease.phase !== 'committed') { + throw new Error('ssh_source_receiving_activation_stale') + } + return settledReceivingActivationLease() + } + if ( + previous && + (activation.clientGeneration <= previous.activation.clientGeneration || + activation.ownerGeneration <= previous.activation.ownerGeneration || + activation.deliveryToken === previous.activation.deliveryToken) + ) { + throw new Error('ssh_source_receiving_activation_stale') + } + return this.installProvisional(relayPtyId, activation, previous) + } + + admit(pending: PendingSshPtySourceData & { source: SshPtySourceFrame }): boolean { + const current = this.deliveryByPty.get(pending.relayPtyId) + if (!acceptsSourceFrame(current, pending.params, pending.source)) { + return false + } + const accepted = Object.freeze({ + ...current, + sourceEndSu: pending.source.sourceEndSu + }) as SourceDeliveryState + this.deliveryByPty.set(pending.relayPtyId, accepted) + if (accepted.lease.phase === 'recovery') { + accepted.lease.recoverySink?.(pending) + } else if (accepted.lease.phase !== 'committed') { + accepted.lease.pendingData.push(pending) + } else { + this.publishData(pending) + } + return true + } + + recordExit(relayPtyId: string): void { + const current = this.deliveryByPty.get(relayPtyId) + if (current?.lease.phase === 'committed') { + this.deliveryByPty.delete(relayPtyId) + } else if (current) { + current.lease.exited = true + } + } + + private installProvisional( + relayPtyId: string, + activation: PtySourceReceivingActivation, + previous: SourceDeliveryState | undefined + ): SshPtySourceDeliveryLease { + const leaseState: SourceDeliveryLeaseState = { + phase: 'provisional', + pendingData: [], + exited: false + } + this.deliveryByPty.set( + relayPtyId, + Object.freeze({ + activation, + sourceEndSu: activation.checkpointSourceEndSu, + lease: leaseState, + ...(previous ? { previous } : {}) + }) + ) + let settled = false + let transferInProgress = false + let rollbackSettlement: Promise | undefined + return Object.freeze({ + commit: () => { + if (settled || transferInProgress) { + return + } + settled = true + this.commit(relayPtyId, leaseState) + }, + rollback: () => { + if (rollbackSettlement) { + return rollbackSettlement + } + if (settled || transferInProgress) { + return Promise.resolve(false) + } + settled = true + this.retire(relayPtyId, previous, leaseState) + rollbackSettlement = settleExactSourceDeliveryCancellation(this.mux, relayPtyId, activation) + return rollbackSettlement + }, + transferToRecovery: (sink) => { + if (settled || transferInProgress || leaseState.phase !== 'provisional') { + throw new Error('ssh_source_receiving_activation_stale') + } + transferInProgress = true + try { + const recoveryLease = this.transferToRecovery(relayPtyId, leaseState, previous, sink) + settled = true + return recoveryLease + } finally { + transferInProgress = false + } + } + }) + } + + private transferToRecovery( + relayPtyId: string, + lease: SourceDeliveryLeaseState, + previous: SourceDeliveryState | undefined, + sink: (pending: PendingSshPtySourceData) => void + ): SshPtyRecoveryActivationLease { + if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) { + this.retire(relayPtyId, previous, lease) + throw new Error('ssh_source_receiving_activation_stale') + } + lease.phase = 'recovery' + lease.recoverySink = sink + try { + while (lease.pendingData.length > 0) { + sink(lease.pendingData.shift()!) + } + } catch (error) { + this.retire(relayPtyId, previous, lease) + throw error + } + let settled = false + return Object.freeze({ + commit: () => { + if (settled) { + return + } + settled = true + lease.recoverySink = undefined + this.commit(relayPtyId, lease) + }, + retire: () => { + if (settled) { + return + } + settled = true + lease.recoverySink = undefined + this.retire(relayPtyId, previous, lease) + } + }) + } + + private commit(relayPtyId: string, lease: SourceDeliveryLeaseState): void { + if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) { + lease.phase = 'retired' + lease.pendingData.splice(0) + return + } + lease.phase = 'committing' + while (lease.pendingData.length > 0) { + this.publishData(lease.pendingData.shift()!) + } + lease.phase = 'committed' + const current = this.deliveryByPty.get(relayPtyId) + if (lease.exited && current?.lease === lease) { + this.deliveryByPty.delete(relayPtyId) + return + } + if (current?.lease === lease && current.previous) { + this.deliveryByPty.set( + relayPtyId, + Object.freeze({ + activation: current.activation, + sourceEndSu: current.sourceEndSu, + lease: current.lease + }) + ) + } + } + + private retire( + relayPtyId: string, + previous: SourceDeliveryState | undefined, + lease: SourceDeliveryLeaseState + ): void { + lease.phase = 'retired' + lease.recoverySink = undefined + lease.pendingData.splice(0) + if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) { + return + } + if (lease.exited) { + this.deliveryByPty.delete(relayPtyId) + return + } + const predecessor = activePredecessor(previous) + if (predecessor) { + this.deliveryByPty.set(relayPtyId, predecessor) + } else { + this.deliveryByPty.delete(relayPtyId) + } + } +} + +function settledReceivingActivationLease(): SshPtySourceDeliveryLease { + return Object.freeze({ + commit: () => {}, + rollback: async () => true, + transferToRecovery: () => Object.freeze({ commit: () => {}, retire: () => {} }) + }) +} + +function activePredecessor(previous?: SourceDeliveryState): SourceDeliveryState | undefined { + while (previous?.lease.phase === 'retired') { + previous = previous.previous + } + return previous +} + +function sameReceivingActivation( + left: PtySourceReceivingActivation, + right: PtySourceReceivingActivation +): boolean { + return ( + left.clientGeneration === right.clientGeneration && + left.ownerGeneration === right.ownerGeneration && + left.ptyIncarnation === right.ptyIncarnation && + left.deliveryToken === right.deliveryToken && + left.checkpointSourceEndSu === right.checkpointSourceEndSu && + left.recoveryEndSu === right.recoveryEndSu + ) +} + +function acceptsSourceFrame( + current: SourceDeliveryState | undefined, + params: Record, + source: SshPtySourceFrame +): current is SourceDeliveryState { + return Boolean( + current && + current.lease.phase !== 'retired' && + !current.lease.exited && + current.activation.ptyIncarnation === params.ptyIncarnation && + current.activation.deliveryToken === source.deliveryToken && + current.activation.clientGeneration === source.clientGeneration && + current.activation.ownerGeneration === source.ownerGeneration && + current.sourceEndSu === source.sourceStartSu + ) +} + +async function settleExactSourceDeliveryCancellation( + mux: SshChannelMultiplexer, + relayPtyId: string, + activation: PtySourceReceivingActivation +): Promise { + try { + const result = (await mux.request('pty.cancelDelivery', { + id: relayPtyId, + clientGeneration: activation.clientGeneration, + ownerGeneration: activation.ownerGeneration, + deliveryToken: activation.deliveryToken + })) as Record + return ( + result.canceled === true && + nonNegativeSafeInteger(result.sentEndSu) && + nonNegativeSafeInteger(result.creditedEndSu) && + result.creditedEndSu <= result.sentEndSu + ) + } catch { + return false + } +} + +function nonNegativeSafeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0 +} diff --git a/src/main/providers/ssh-pty-source-frame.ts b/src/main/providers/ssh-pty-source-frame.ts new file mode 100644 index 00000000000..2165b4ba7c4 --- /dev/null +++ b/src/main/providers/ssh-pty-source-frame.ts @@ -0,0 +1,64 @@ +import type { SshPtyDataCallback } from './ssh-pty-provider-contract' + +export type SshPtySourceFrame = NonNullable[0]['source']> + +const SOURCE_KEYS = [ + 'deliveryToken', + 'clientGeneration', + 'ownerGeneration', + 'sourceEndSu', + 'sourceLengthSu', + 'ptyIncarnation' +] as const + +export function parseSshPtySourceFrame( + params: Record, + data: string, + relayPtyId: string +): Readonly<{ source?: SshPtySourceFrame; malformed: boolean }> { + if (!SOURCE_KEYS.some((key) => params[key] !== undefined)) { + return Object.freeze({ malformed: false }) + } + const sourceEndSu = params.sourceEndSu + const sourceLengthSu = params.sourceLengthSu + const rawLength = params.rawLength + const transformed = params.transformed === true + if ( + typeof params.data !== 'string' || + typeof params.deliveryToken !== 'string' || + params.deliveryToken.length === 0 || + !positiveSafeInteger(params.clientGeneration) || + !positiveSafeInteger(params.ownerGeneration) || + !nonNegativeSafeInteger(sourceEndSu) || + !nonNegativeSafeInteger(sourceLengthSu) || + sourceEndSu < sourceLengthSu || + typeof params.ptyIncarnation !== 'string' || + params.ptyIncarnation.length === 0 || + (params.seq !== undefined && !nonNegativeSafeInteger(params.seq)) || + (transformed ? rawLength !== sourceLengthSu : data.length !== sourceLengthSu) || + (rawLength !== undefined && rawLength !== sourceLengthSu) + ) { + return Object.freeze({ malformed: true }) + } + const sourceStartSu = sourceEndSu - sourceLengthSu + return Object.freeze({ + malformed: false, + source: Object.freeze({ + relayPtyId, + spanId: `${params.deliveryToken}:${sourceStartSu}:${sourceEndSu}`, + clientGeneration: params.clientGeneration, + ownerGeneration: params.ownerGeneration, + deliveryToken: params.deliveryToken, + sourceStartSu, + sourceEndSu + }) + }) +} + +function positiveSafeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) > 0 +} + +function nonNegativeSafeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0 +} diff --git a/src/main/providers/ssh-pty-spawn-env.ts b/src/main/providers/ssh-pty-spawn-env.ts index ad9cc7377a4..595e750c90d 100644 --- a/src/main/providers/ssh-pty-spawn-env.ts +++ b/src/main/providers/ssh-pty-spawn-env.ts @@ -22,6 +22,9 @@ export function buildSshPtySpawnEnv(args: { merged.ORCA_RELAY_DIR = args.remoteCliBridgeEnv.relayDir merged.ORCA_RELAY_NODE_PATH = args.remoteCliBridgeEnv.nodePath merged.ORCA_RELAY_SOCKET_PATH = args.remoteCliBridgeEnv.sockPath + if (args.remoteCliBridgeEnv.credentialFile) { + merged.ORCA_RELAY_CREDENTIAL_FILE = args.remoteCliBridgeEnv.credentialFile + } } // Why: match local/daemon precedence—managed defaults cannot restore explicitly removed values. for (const key of args.envToDelete ?? []) { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index c41d88c83a5..9cee18d16cf 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -16,6 +16,13 @@ import { parseFileUriPathParts } from '../daemon/osc7-file-uri' import type { AgentStatus } from '../../shared/agent-detection' import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' import type { TerminalOscColorQueryReplyColors } from '../../shared/terminal-osc-color-reply' +import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range' +import type { + RemoteTerminalSourceRangeConsumerHooks, + RemoteTerminalSourceRangeReplacementPublication, + RemoteTerminalSourceRangeReplacementReservation, + RemoteTerminalSourceRangeStreamIdentity +} from './remote-terminal-source-range-consumer' import { createTerminalTitleTracker, stripBrailleSpinnerGlyphs, @@ -1431,6 +1438,19 @@ type RuntimeHeadlessTerminal = { writeChain: Promise } +export type RuntimePtyDataAdmission = Readonly<{ + sequence: number + completion: Promise +}> + +export type RuntimeTerminalDataMeta = Readonly<{ + seq?: number + rawLength?: number + transformed?: boolean + cwd?: string + sourceRanges?: readonly TerminalOutputSourceRange[] +}> + type RuntimeVisibleTerminalState = { lines: string[] isAlternateScreen: boolean @@ -2669,13 +2689,10 @@ export class OrcaRuntimeService { // without polling. Keyed by ptyId for O(1) lookup per data event. private dataListeners = new Map< string, - Set< - ( - data: string, - meta?: { seq?: number; rawLength?: number; transformed?: boolean; cwd?: string } - ) => void - > + Set<(data: string, meta?: RuntimeTerminalDataMeta) => void> >() + private remoteTerminalSourceRangeConsumerHooks: RemoteTerminalSourceRangeConsumerHooks | null = + null // Why: startup draft paste can subscribe after the agent already emitted its // ready marker. Keep a bounded raw buffer so fast startup output is replayed. private recentPtyOutputById = new Map() @@ -8616,16 +8633,49 @@ export class OrcaRuntimeService { } } + resetPtyModelAfterMigrationFailure(ptyId: string): void { + this.providerSnapshotPreferredPtys.add(ptyId) + this.disposeHeadlessTerminal(ptyId) + } + /** * Handles incoming data from a PTY process, running agent detection, * updating terminal tail buffers, and triggering foreground agent refreshes. */ + acceptPtyDataBounded( + ptyId: string, + data: string, + at: number, + sequenceChars = data.length, + transformed = false, + sourceRanges?: readonly TerminalOutputSourceRange[] + ): RuntimePtyDataAdmission { + let completion: Promise | null = null + const sequence = this.onPtyData( + ptyId, + data, + at, + sequenceChars, + transformed, + (receipt) => { + completion = receipt + }, + sourceRanges + ) + if (!completion) { + throw new Error('PTY model admission receipt was not captured') + } + return Object.freeze({ sequence, completion }) + } + onPtyData( ptyId: string, data: string, at: number, sequenceChars = data.length, - transformed = false + transformed = false, + captureModelReceipt?: (completion: Promise) => void, + sourceRanges?: readonly TerminalOutputSourceRange[] ): number { const outputSequence = (this.ptyOutputSequenceById.get(ptyId) ?? 0) + sequenceChars this.ptyOutputSequenceById.set(ptyId, outputSequence) @@ -8664,7 +8714,13 @@ export class OrcaRuntimeService { // applyTrackedPtyTitle) in byte order, superseding main's inline // extractLastOscTitleForPty block (#7880/#7852 title/status semantics are // preserved via the tracker + detectAgentStatusFromTitle path). - this.trackHeadlessTerminalData(ptyId, data, outputSequence, forwardQueryReplies) + const modelCompletion = this.trackHeadlessTerminalData( + ptyId, + data, + outputSequence, + forwardQueryReplies + ) + captureModelReceipt?.(modelCompletion) const pty = this.getOrCreatePtyWorktreeRecord(ptyId) const ptyTailBefore = pty @@ -8846,7 +8902,8 @@ export class OrcaRuntimeService { seq: outputSequence, rawLength: sequenceChars, ...(transformed ? { transformed: true } : {}), - ...(cwdChanged && cwd !== null ? { cwd } : {}) + ...(cwdChanged && cwd !== null ? { cwd } : {}), + ...(sourceRanges && sourceRanges.length > 0 ? { sourceRanges } : {}) } for (const listener of listeners) { try { @@ -9721,14 +9778,71 @@ export class OrcaRuntimeService { subscribeToTerminalData( ptyId: string, - listener: ( - data: string, - meta?: { seq?: number; rawLength?: number; transformed?: boolean; cwd?: string } - ) => void + listener: (data: string, meta?: RuntimeTerminalDataMeta) => void ): () => void { return addListenerToMap(this.dataListeners, ptyId, listener) } + setRemoteTerminalSourceRangeConsumerHooks( + hooks: RemoteTerminalSourceRangeConsumerHooks | null + ): void { + this.remoteTerminalSourceRangeConsumerHooks = hooks + } + + attachRemoteTerminalSourceRangeConsumer( + identity: RemoteTerminalSourceRangeStreamIdentity + ): boolean { + return this.remoteTerminalSourceRangeConsumerHooks?.attach(identity) ?? false + } + + settleRemoteTerminalSourceRanges( + identity: RemoteTerminalSourceRangeStreamIdentity, + ranges: readonly TerminalOutputSourceRange[] + ): void { + this.remoteTerminalSourceRangeConsumerHooks?.settle(identity, ranges) + } + + reserveRemoteTerminalSourceRangeReplacement( + identity: RemoteTerminalSourceRangeStreamIdentity, + requiredSeq: number, + reason: string + ): RemoteTerminalSourceRangeReplacementReservation | null { + return ( + this.remoteTerminalSourceRangeConsumerHooks?.reserveReplacement( + identity, + requiredSeq, + reason + ) ?? null + ) + } + + commitRemoteTerminalSourceRangeReplacement( + reservation: RemoteTerminalSourceRangeReplacementReservation, + publication: RemoteTerminalSourceRangeReplacementPublication + ): boolean { + return ( + this.remoteTerminalSourceRangeConsumerHooks?.commitReplacement(reservation, publication) ?? + false + ) + } + + rollbackRemoteTerminalSourceRangeReplacement( + reservation: RemoteTerminalSourceRangeReplacementReservation, + reason: string + ): boolean { + return ( + this.remoteTerminalSourceRangeConsumerHooks?.rollbackReplacement(reservation, reason) ?? false + ) + } + + cancelRemoteTerminalSourceRanges( + identity: RemoteTerminalSourceRangeStreamIdentity, + ranges: readonly TerminalOutputSourceRange[], + reason: string + ): void { + this.remoteTerminalSourceRangeConsumerHooks?.cancel(identity, ranges, reason) + } + /** Set by pty IPC: fires when a PTY gains/loses remote view subscribers so * the daemon background mark (keep-tail stream thinning) can resync — a * live mobile/web view consumes raw bytes and must never be thinned, even @@ -10186,19 +10300,17 @@ export class OrcaRuntimeService { data: string, outputSequence: number, forwardQueryReplies = false - ): void { + ): Promise { const state = this.getOrCreateHeadlessTerminal(ptyId) - state.writeChain = state.writeChain - .then(async () => { - // Why: the ingestion-time ownership decision is closed over this - // chain link; async scheduling cannot retroactively change it. - await state.emulator.write(data, { forwardQueryReplies }) - state.outputSequence = outputSequence - }) - .catch(() => { - // Best-effort state tracking; live streaming must continue even if - // xterm rejects a malformed or raced write during shutdown. - }) + const completion = state.writeChain.then(async () => { + // Why: the ingestion-time ownership decision is closed over this + // chain link; async scheduling cannot retroactively change it. + await state.emulator.write(data, { forwardQueryReplies }) + state.outputSequence = outputSequence + }) + // Legacy callers remain best-effort; bounded SSH admission observes the raw receipt. + state.writeChain = completion.catch(() => {}) + return completion } /** Shared factory for the per-PTY runtime emulators (seed, hydration, and diff --git a/src/main/runtime/remote-terminal-source-range-consumer.ts b/src/main/runtime/remote-terminal-source-range-consumer.ts new file mode 100644 index 00000000000..3924d19bfcd --- /dev/null +++ b/src/main/runtime/remote-terminal-source-range-consumer.ts @@ -0,0 +1,44 @@ +import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range' + +export type RemoteTerminalSourceRangeStreamIdentity = Readonly<{ + ptyId: string + consumerId: string + streamGeneration: string +}> + +export type RemoteTerminalSourceRangeReplacementReservation = Readonly<{ + reservationId: string + identity: RemoteTerminalSourceRangeStreamIdentity + requiredSeq: number +}> + +export type RemoteTerminalSourceRangeReplacementPublication = Readonly<{ + source: 'headless' | 'renderer' + seq: number +}> + +export type RemoteTerminalSourceRangeConsumerHooks = { + attach: (identity: RemoteTerminalSourceRangeStreamIdentity) => boolean + settle: ( + identity: RemoteTerminalSourceRangeStreamIdentity, + ranges: readonly TerminalOutputSourceRange[] + ) => void + reserveReplacement: ( + identity: RemoteTerminalSourceRangeStreamIdentity, + requiredSeq: number, + reason: string + ) => RemoteTerminalSourceRangeReplacementReservation | null + commitReplacement: ( + reservation: RemoteTerminalSourceRangeReplacementReservation, + publication: RemoteTerminalSourceRangeReplacementPublication + ) => boolean + rollbackReplacement: ( + reservation: RemoteTerminalSourceRangeReplacementReservation, + reason: string + ) => boolean + cancel: ( + identity: RemoteTerminalSourceRangeStreamIdentity, + ranges: readonly TerminalOutputSourceRange[], + reason: string + ) => void +} diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index 4e4801c9faa..2b5d65a5d04 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -1,5 +1,6 @@ /* oxlint-disable max-lines -- Why: terminal RPC methods are co-located for discoverability; splitting would scatter related handlers across files. */ import { z } from 'zod' +import { randomUUID } from 'node:crypto' import { InvalidArgumentError, defineMethod, @@ -19,6 +20,7 @@ import { } from '../../../../shared/terminal-stream-protocol' import { iterateTerminalOutputFrameChunks, + sliceTerminalOutputSourceRanges, type TerminalOutputFrameChunk, type TerminalOutputMeta } from '../terminal-output-frame-chunks' @@ -61,6 +63,13 @@ import { TERMINAL_OUTPUT_BATCH_MAX_BYTES } from '../../../../shared/terminal-multiplex-flow-control' import { drainTerminalMultiplexRoundRobin } from '../terminal-multiplex-round-robin' +import type { TerminalSourceRangeLedger } from '../terminal-source-range-ledger' +import { TerminalSourceRangeRegistry } from '../terminal-source-range-registry' +import { + sameTerminalOutputSourceIdentity, + type TerminalOutputSourceRange +} from '../../../../shared/terminal-output-source-range' +import type { RemoteTerminalSourceRangeReplacementReservation } from '../../remote-terminal-source-range-consumer' const REQUESTED_SNAPSHOT_BYTE_BUDGET = 2 * 1024 * 1024 const TERMINAL_OUTPUT_FLUSH_MS = 5 @@ -112,6 +121,11 @@ type TerminalMultiplexStream = { client: TerminalViewportClient | undefined isMobile: boolean ackOutput: boolean + ackOutputSourceRanges: boolean + streamGeneration: string + sourceRangeLedger: TerminalSourceRangeLedger | null + sourceRangeConsumerAttached: boolean + sourceRangeReplacement: RemoteTerminalSourceRangeReplacementReservation | null ackInFlightBytes: number ackWindowBytes: number supportsDesktopViewportClaims: boolean @@ -157,6 +171,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp let lastSeq: number | undefined let pendingCwd: string | undefined let pendingRawLength = 0 + let pendingSourceRanges: TerminalOutputSourceRange[] = [] let timer: ReturnType | null = null const clearTimer = (): void => { @@ -174,10 +189,13 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp } const data = chunks.length === 1 ? chunks[0]! : chunks.join('') const meta = - typeof lastSeq === 'number' || pendingCwd !== undefined + typeof lastSeq === 'number' || pendingCwd !== undefined || pendingSourceRanges.length > 0 ? { ...(typeof lastSeq === 'number' ? { seq: lastSeq, rawLength: pendingRawLength } : {}), - ...(pendingCwd !== undefined ? { cwd: pendingCwd } : {}) + ...(pendingCwd !== undefined ? { cwd: pendingCwd } : {}), + ...(pendingSourceRanges.length > 0 + ? { sourceRanges: Object.freeze(pendingSourceRanges.slice()) } + : {}) } : undefined chunks = [] @@ -185,6 +203,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp lastSeq = undefined pendingCwd = undefined pendingRawLength = 0 + pendingSourceRanges = [] onFlush(data, meta) } @@ -199,12 +218,26 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp onFlush(data, { ...meta, rawLength, transformed: true }) return } + const nextSourceRanges = meta?.sourceRanges ?? [] + const lastSourceRange = pendingSourceRanges.at(-1) + const firstNextSourceRange = nextSourceRanges[0] + if ( + chunks.length > 0 && + (pendingSourceRanges.length > 0 !== nextSourceRanges.length > 0 || + (lastSourceRange && + firstNextSourceRange && + (!sameTerminalOutputSourceIdentity(lastSourceRange, firstNextSourceRange) || + lastSourceRange.displayEnd !== firstNextSourceRange.displayStart))) + ) { + flush() + } if (meta?.cwd !== undefined) { flush() pendingCwd = meta.cwd } chunks.push(data) pendingRawLength += rawLength + pendingSourceRanges.push(...nextSourceRanges) const remainingBudget = Math.max(1, TERMINAL_OUTPUT_BATCH_MAX_BYTES - bytes) const measurement = measureTerminalStreamByteLength(data, { stopAfterBytes: remainingBudget @@ -231,6 +264,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp chunks = [] bytes = 0 pendingRawLength = 0 + pendingSourceRanges = [] } } } @@ -380,22 +414,38 @@ function appendPendingMultiplexOutput( function getOutputAfterSnapshotSeq( chunk: TerminalOutputChunk, snapshotSeq: number | undefined -): string | null { +): TerminalOutputChunk | null { if ( typeof snapshotSeq !== 'number' || typeof chunk.meta?.seq !== 'number' || typeof chunk.meta.rawLength !== 'number' ) { - return chunk.data + return chunk } if (chunk.meta.seq <= snapshotSeq) { return null } const chunkStartSeq = chunk.meta.seq - chunk.meta.rawLength if (chunkStartSeq >= snapshotSeq) { - return chunk.data + return chunk + } + if (chunk.meta.transformed) { + return null + } + const offset = snapshotSeq - chunkStartSeq + return { + data: chunk.data.slice(offset), + bytes: chunk.bytes, + meta: { + ...chunk.meta, + rawLength: chunk.meta.rawLength - offset, + sourceRanges: sliceTerminalOutputSourceRanges( + chunk.meta.sourceRanges, + offset, + chunk.data.length + ) + } } - return chunk.data.slice(snapshotSeq - chunkStartSeq) } function stripSnapshotBoundaryQuerySuffixes( @@ -551,36 +601,45 @@ async function serializeBudgetedRequestedSnapshot( } function sendSnapshotFrames( - sendFrame: (opcode: TerminalStreamOpcode, payload?: Uint8Array) => void, + sendFrame: ( + opcode: TerminalStreamOpcode, + payload?: Uint8Array + ) => boolean | void, options: SnapshotFrameOptions -): { bytes: number; chunks: number } { - sendFrame( - TerminalStreamOpcode.SnapshotStart, - encodeTerminalStreamJson({ - kind: options.kind, - cols: options.cols, - rows: options.rows, - requestId: options.requestId, - displayMode: options.displayMode, - reason: options.reason, - seq: options.seq, - cwd: options.cwd, - source: options.source, - oscLinks: options.oscLinks, - pendingEscapeTailAnsi: options.pendingEscapeTailAnsi, - truncated: options.truncated === true, - truncatedByByteBudget: options.truncatedByByteBudget === true - }) - ) +): { bytes: number; chunks: number; published: boolean } { + if ( + sendFrame( + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + kind: options.kind, + cols: options.cols, + rows: options.rows, + requestId: options.requestId, + displayMode: options.displayMode, + reason: options.reason, + seq: options.seq, + cwd: options.cwd, + source: options.source, + oscLinks: options.oscLinks, + pendingEscapeTailAnsi: options.pendingEscapeTailAnsi, + truncated: options.truncated === true, + truncatedByByteBudget: options.truncatedByByteBudget === true + }) + ) === false + ) { + return { bytes: 0, chunks: 0, published: false } + } let chunks = 0 let bytes = 0 for (const chunk of iterateTerminalStreamTextPayloads(options.data)) { + if (sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk) === false) { + return { bytes, chunks, published: false } + } chunks++ bytes += chunk.byteLength - sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk) } - sendFrame(TerminalStreamOpcode.SnapshotEnd) - return { bytes, chunks } + const published = sendFrame(TerminalStreamOpcode.SnapshotEnd) !== false + return { bytes, chunks, published } } async function serializeBudgetedMobileSnapshot( @@ -953,14 +1012,24 @@ const TerminalMultiplexSubscribeFrame = TerminalHandle.extend({ capabilities: z .object({ ackOutput: z.literal(1).optional(), + ackOutputSourceRanges: z.literal(1).optional(), desktopViewportClaims: z.literal(1).optional() }) .optional() }) -const TerminalMultiplexAckFrame = z.object({ - bytes: z.number().int().nonnegative() -}) +const TerminalMultiplexLegacyAckFrame = z + .object({ + bytes: z.number().int().nonnegative() + }) + .strict() + +const TerminalMultiplexSourceRangeAckFrame = z + .object({ + streamGeneration: z.string().min(1), + ackedEndByte: z.number().int().nonnegative() + }) + .strict() const TerminalMultiplexSnapshotRequestFrame = z.object({ requestId: z.number().int().positive().optional(), @@ -1514,6 +1583,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ let closed = false let cursor = 0 const streams = new Map() + const sourceRangeRegistry = new TerminalSourceRangeRegistry() const pendingPtyWaitControllers = new Map>() let ackTotalInFlightBytes = 0 let ackTotalWindowBytes = TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES @@ -1526,9 +1596,11 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ streamId: number, opcode: TerminalStreamOpcode, payload: Uint8Array = new Uint8Array(), - seq?: number + seq?: number, + onRejected?: () => void ): boolean => { if (closed) { + onRejected?.() return false } // Why: a seq-less Output chunk must carry sentinel 0, not the control-frame cursor, or it poisons the client's frame-drop tracker. @@ -1540,10 +1612,12 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ encodeTerminalStreamFrame({ opcode, streamId, seq: resolvedSeq, payload }) ) } catch { + onRejected?.() closeMultiplex() return false } if (sent === false) { + onRejected?.() // Why: false means the transport discarded this frame; reconnect is the only available retry boundary with an authoritative snapshot. closeMultiplex() return false @@ -1577,22 +1651,43 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ } return ( stream.ackInFlightBytes + bytes <= stream.ackWindowBytes && - ackTotalInFlightBytes + bytes <= ackTotalWindowBytes + ackTotalInFlightBytes + bytes <= ackTotalWindowBytes && + (!stream.ackOutputSourceRanges || stream.sourceRangeLedger?.canAccept(bytes) === true) ) } const sendAckGatedOutput = ( stream: TerminalMultiplexStream, chunk: TerminalOutputFrameChunk ): boolean => { + const prepared = stream.ackOutputSourceRanges + ? stream.sourceRangeLedger?.prepareAccept( + chunk.bytes.byteLength, + chunk.displayLength, + chunk.sourceRanges ?? [], + chunk.seq + ) + : undefined + if (stream.ackOutputSourceRanges && prepared?.status !== 'ready') { + if (prepared?.status !== 'capacity') { + detachStream(stream.streamId, true) + } + return false + } + const admission = prepared?.status === 'ready' ? prepared.admission : undefined const sent = sendFrame( stream.streamId, chunk.opcode ?? TerminalStreamOpcode.Output, chunk.bytes, - chunk.seq + chunk.seq, + admission?.rollback ) if (!sent) { return false } + if (admission && !admission.commit()) { + detachStream(stream.streamId, true) + return false + } if (stream.ackOutput) { stream.ackInFlightBytes += chunk.bytes.byteLength ackTotalInFlightBytes += chunk.bytes.byteLength @@ -1625,6 +1720,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ return } stream.ackRecoverySnapshotInFlight = true + let replacement: RemoteTerminalSourceRangeReplacementReservation | null = null try { const serialized = await serializeBudgetedRequestedSnapshot(runtime, stream.ptyId, 0) if (closed || streams.get(stream.streamId) !== stream) { @@ -1633,21 +1729,75 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ if (!serialized) { throw new Error('Remote terminal recovery snapshot unavailable.') } + if ( + stream.ackOutputSourceRanges && + (serialized.source === undefined || typeof serialized.seq !== 'number') + ) { + throw new Error('Remote terminal recovery snapshot source identity unavailable.') + } + if ( + stream.ackOutputSourceRanges && + serialized.source !== undefined && + typeof serialized.seq === 'number' + ) { + replacement = runtime.reserveRemoteTerminalSourceRangeReplacement( + { + ptyId: stream.ptyId, + consumerId: stream.remoteDesktopSubscriptionKey, + streamGeneration: stream.streamGeneration + }, + serialized.seq, + 'ack-pending-overflow' + ) + stream.sourceRangeReplacement = replacement + } const displayMode = runtime.getMobileDisplayMode(stream.ptyId) - // Why: dropped ACK-pending output breaks live replay; send a fresh snapshot before resuming output. - sendSnapshotFrames((opcode, payload) => sendFrame(stream.streamId, opcode, payload), { - kind: 'scrollback', - cols: serialized.cols, - rows: serialized.rows, - displayMode, - reason: 'ack-pending-overflow', - seq: serialized.seq, - source: serialized.source, - truncatedByByteBudget: serialized.truncatedByByteBudget, - data: serialized.data - }) + const publication = sendSnapshotFrames( + (opcode, payload) => + !closed && + streams.get(stream.streamId) === stream && + sendFrame(stream.streamId, opcode, payload), + { + kind: 'scrollback', + cols: serialized.cols, + rows: serialized.rows, + displayMode, + reason: 'ack-pending-overflow', + seq: serialized.seq, + source: serialized.source, + truncatedByByteBudget: serialized.truncatedByByteBudget, + data: serialized.data + } + ) + if (!publication.published) { + throw new Error('Remote terminal recovery snapshot was not published.') + } + if (closed || streams.get(stream.streamId) !== stream) { + throw new Error('Remote terminal recovery snapshot stream detached.') + } + const localReplacement = replacement + ? typeof serialized.seq === 'number' + ? stream.sourceRangeLedger?.planSourceRangeReplacement(serialized.seq) + : null + : null + if (replacement && !localReplacement) { + throw new Error('Remote terminal recovery source ledger replacement unavailable.') + } + if ( + replacement && + (!serialized.source || + typeof serialized.seq !== 'number' || + !runtime.commitRemoteTerminalSourceRangeReplacement(replacement, { + source: serialized.source, + seq: serialized.seq + })) + ) { + throw new Error('Remote terminal recovery snapshot replacement was not accepted.') + } + localReplacement?.commit() + stream.sourceRangeReplacement = null + replacement = null if (typeof serialized.seq === 'number') { - // Why: chunks queued before the snapshot serialized are already in it; replaying them would duplicate output. const snapshotSeq = serialized.seq const retained = stream.ackPendingOutput.filter( (chunk) => !(typeof chunk.seq === 'number' && chunk.seq <= snapshotSeq) @@ -1660,11 +1810,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ } stream.ackPendingOutputOverflowed = false } catch (error) { + if (replacement) { + if (stream.sourceRangeReplacement === replacement) { + stream.sourceRangeReplacement = null + runtime.rollbackRemoteTerminalSourceRangeReplacement( + replacement, + 'ack-pending-overflow-unpublished' + ) + } + replacement = null + } + if (closed || streams.get(stream.streamId) !== stream) { + return + } sendStreamError( stream.streamId, error instanceof Error ? error.message : 'Remote terminal recovery snapshot failed.' ) - // Why: retrying the same failed recovery from finally creates an unbounded error loop. detachStream(stream.streamId, true) } finally { if (streams.get(stream.streamId) === stream) { @@ -1735,6 +1897,56 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - acknowledged) flushAllAckPendingOutput() } + const acknowledgeSourceRanges = ( + stream: TerminalMultiplexStream, + streamGeneration: string, + ackedEndByte: number + ): void => { + if (!stream.ackOutputSourceRanges) { + return + } + const result = stream.sourceRangeLedger?.acknowledge(streamGeneration, ackedEndByte) + if (!result) { + return + } + if (result.status !== 'accepted') { + return + } + if (result.settled.length > 0) { + runtime.settleRemoteTerminalSourceRanges( + { + ptyId: stream.ptyId, + consumerId: stream.remoteDesktopSubscriptionKey, + streamGeneration: stream.streamGeneration + }, + result.settled + ) + } + acknowledgeOutput(stream, result.acknowledgedBytes) + } + const detachSourceRangeConsumer = (stream: TerminalMultiplexStream, reason: string): void => { + if (!stream.sourceRangeConsumerAttached) { + return + } + stream.sourceRangeConsumerAttached = false + const ledger = stream.sourceRangeLedger + stream.sourceRangeLedger = null + if (!ledger) { + return + } + const identity = { + ptyId: stream.ptyId, + consumerId: stream.remoteDesktopSubscriptionKey, + streamGeneration: stream.streamGeneration + } + const transfer = ledger.beginTransfer() + const ranges = transfer.frames.flatMap((frame) => frame.sourceRanges) + try { + runtime.cancelRemoteTerminalSourceRanges(identity, ranges, reason) + } finally { + transfer.commit() + } + } const detachStream = ( streamId: number, emitEnd: boolean, @@ -1744,8 +1956,17 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ if (!stream) { return } + const replacement = stream.sourceRangeReplacement + stream.sourceRangeReplacement = null + if (replacement) { + runtime.rollbackRemoteTerminalSourceRangeReplacement( + replacement, + 'stream-detached-replacement-aborted' + ) + } stream.outputBatcher.flush() stream.outputBatcher.dispose() + detachSourceRangeConsumer(stream, 'stream-detached') ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - stream.ackInFlightBytes) stream.ackInFlightBytes = 0 stream.ackPendingOutput = [] @@ -1827,11 +2048,21 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ return } if (frame.opcode === TerminalStreamOpcode.Ack) { - const parsed = TerminalMultiplexAckFrame.safeParse( - decodeTerminalStreamJson(frame.payload) ?? {} - ) - if (parsed.success) { - acknowledgeOutput(stream, parsed.data.bytes) + const payload = decodeTerminalStreamJson(frame.payload) ?? {} + if (stream.ackOutputSourceRanges) { + const parsed = TerminalMultiplexSourceRangeAckFrame.safeParse(payload) + if (parsed.success) { + acknowledgeSourceRanges( + stream, + parsed.data.streamGeneration, + parsed.data.ackedEndByte + ) + } + } else { + const parsed = TerminalMultiplexLegacyAckFrame.safeParse(payload) + if (parsed.success) { + acknowledgeOutput(stream, parsed.data.bytes) + } } return } @@ -2024,12 +2255,12 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ // high-water, so covered bytes would render twice; tagged // snapshots feed a side consumer and the live view still // needs every buffered chunk. - const uncoveredData = + const uncovered = typeof requestId === 'number' - ? chunk.data + ? chunk : getOutputAfterSnapshotSeq(chunk, sentSnapshotOutputSeq) - if (uncoveredData) { - stream.outputBatcher.push(uncoveredData, chunk.meta) + if (uncovered) { + stream.outputBatcher.push(uncovered.data, uncovered.meta) } } } @@ -2140,6 +2371,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ detachStream(request.streamId, false) const ptyId = leaf.ptyId + const remoteDesktopSubscriptionKey = `multiplex:${connectionId}:${request.streamId}` + const streamGeneration = randomUUID() + const requestedSourceRangeConsumer = + request.capabilities?.ackOutput === 1 && request.capabilities?.ackOutputSourceRanges === 1 + const sourceRangeLedger = requestedSourceRangeConsumer + ? sourceRangeRegistry.open(streamGeneration) + : null + const sourceRangeConsumerAttached = + sourceRangeLedger !== null && + runtime.attachRemoteTerminalSourceRangeConsumer({ + ptyId, + consumerId: remoteDesktopSubscriptionKey, + streamGeneration + }) + if (!sourceRangeConsumerAttached) { + sourceRangeLedger?.close() + } const stream: TerminalMultiplexStream = { streamId: request.streamId, terminal: request.terminal, @@ -2147,13 +2395,18 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ client: request.client, isMobile, ackOutput: request.capabilities?.ackOutput === 1, + ackOutputSourceRanges: sourceRangeConsumerAttached, + streamGeneration, + sourceRangeLedger: sourceRangeConsumerAttached ? sourceRangeLedger : null, + sourceRangeConsumerAttached, + sourceRangeReplacement: null, ackInFlightBytes: 0, ackWindowBytes: TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES, supportsDesktopViewportClaims: request.capabilities?.desktopViewportClaims === 1, desktopClaimTail: Promise.resolve(true), registeredRemoteDesktopDriver: false, // Why: streamId is client-local, so key the width floor by connectionId or two connections sharing stream 1 for one PTY clobber each other's floor. - remoteDesktopSubscriptionKey: `multiplex:${connectionId}:${request.streamId}`, + remoteDesktopSubscriptionKey, pendingRemoteDesktopViewport: null, buffering: true, ackPendingOutput: [], @@ -2273,35 +2526,77 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ rows: serialized?.rows ?? size?.rows, displayMode, seq: layoutSeq, + ...(stream.ackOutputSourceRanges + ? { + streamGeneration: stream.streamGeneration, + capabilities: { ackOutputSourceRanges: 1 as const } + } + : {}), truncated: initialOutputOverflowed || (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)) }) - sendSnapshotFrames((opcode, payload) => sendFrame(request.streamId, opcode, payload), { - kind: 'scrollback', - cols: serialized?.cols ?? size?.cols ?? 80, - rows: serialized?.rows ?? size?.rows ?? 24, - displayMode, - seq: snapshotFrameSeq, - cwd: serialized?.cwd, - truncated: - initialOutputOverflowed || - (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), - truncatedByByteBudget: serialized?.truncatedByByteBudget, - source: serialized?.source, - oscLinks: serialized?.oscLinks, - pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi, - data: serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '') - }) + stream.sourceRangeReplacement = + stream.ackOutputSourceRanges && + serialized?.source !== undefined && + typeof serialized.seq === 'number' + ? runtime.reserveRemoteTerminalSourceRangeReplacement( + { + ptyId, + consumerId: stream.remoteDesktopSubscriptionKey, + streamGeneration: stream.streamGeneration + }, + serialized.seq, + 'initial-snapshot' + ) + : null + const snapshotPublication = sendSnapshotFrames( + (opcode, payload) => sendFrame(request.streamId, opcode, payload), + { + kind: 'scrollback', + cols: serialized?.cols ?? size?.cols ?? 80, + rows: serialized?.rows ?? size?.rows ?? 24, + displayMode, + seq: snapshotFrameSeq, + cwd: serialized?.cwd, + truncated: + initialOutputOverflowed || + (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), + truncatedByByteBudget: serialized?.truncatedByByteBudget, + source: serialized?.source, + oscLinks: serialized?.oscLinks, + pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi, + data: + serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '') + } + ) + const replacement = stream.sourceRangeReplacement + stream.sourceRangeReplacement = null + if (replacement) { + const committed = + snapshotPublication.published && + serialized?.source !== undefined && + typeof serialized.seq === 'number' && + runtime.commitRemoteTerminalSourceRangeReplacement(replacement, { + source: serialized.source, + seq: serialized.seq + }) + if (!committed) { + runtime.rollbackRemoteTerminalSourceRangeReplacement( + replacement, + 'initial-snapshot-unpublished' + ) + } + } // Why: baseline for resize re-stream gating; the client already rewrapped to these cols via the initial snapshot replay. stream.lastResizeCols = serialized?.cols ?? size?.cols stream.buffering = false const pendingOutput = stream.pendingOutput.splice(0) if (!initialOutputOverflowed) { for (const chunk of pendingOutput) { - const uncoveredData = getOutputAfterSnapshotSeq(chunk, snapshotOutputSeq) - if (uncoveredData) { - stream.outputBatcher.push(uncoveredData, chunk.meta) + const uncovered = getOutputAfterSnapshotSeq(chunk, snapshotOutputSeq) + if (uncovered) { + stream.outputBatcher.push(uncovered.data, uncovered.meta) } } } @@ -2982,9 +3277,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ if (stableRendererSnapshot?.data.length) { serialized = stableRendererSnapshot const trailingOutput = pendingOutput.flatMap((item) => { - const data = getOutputAfterSnapshotSeq(item, stableRendererSnapshot.seq) + const output = getOutputAfterSnapshotSeq(item, stableRendererSnapshot.seq) const seq = item.meta?.seq - return data && typeof seq === 'number' ? [{ data, seq }] : [] + return output && typeof seq === 'number' ? [{ data: output.data, seq }] : [] }) runtime.replaceHeadlessTerminalFromRendererSnapshotForRecovery( ptyId, @@ -3118,8 +3413,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ } if (!initialOutputOverflowed) { for (const item of bufferedOutput) { - let uncoveredData = getOutputAfterSnapshotSeq(item, snapshotOutputSeq) - let uncoveredMeta = item.meta + const uncovered = getOutputAfterSnapshotSeq(item, snapshotOutputSeq) + let uncoveredData = uncovered?.data ?? null + let uncoveredMeta = uncovered?.meta if ( uncoveredData && uncoveredData !== item.data && diff --git a/src/main/runtime/rpc/terminal-multiplex.test.ts b/src/main/runtime/rpc/terminal-multiplex.test.ts index d86161d0321..0779ea370c7 100644 --- a/src/main/runtime/rpc/terminal-multiplex.test.ts +++ b/src/main/runtime/rpc/terminal-multiplex.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { RpcDispatcher } from './dispatcher' import type { RpcRequest } from './core' -import type { OrcaRuntimeService } from '../orca-runtime' +import type { OrcaRuntimeService, RuntimeTerminalDataMeta } from '../orca-runtime' import { TERMINAL_METHODS } from './methods/terminal' import type { RuntimeTerminalWait } from '../../../shared/runtime-types' import { @@ -14,6 +14,8 @@ import { encodeTerminalStreamJson, encodeTerminalStreamText } from '../../../shared/terminal-stream-protocol' +import { TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES } from '../../../shared/terminal-multiplex-flow-control' +import { SshPtyOutputIntake, type SshPtyOutputDataEvent } from '../../ipc/ssh-pty-output-intake' function stubRuntime(overrides: Partial = {}): OrcaRuntimeService { return { @@ -33,6 +35,7 @@ function stubRuntime(overrides: Partial = {}): OrcaRuntimeSe isPtyResizeDrivenRemotely: vi.fn().mockReturnValue(false), getRemoteDesktopFitHold: vi.fn().mockReturnValue({ mode: 'desktop-fit', cols: 120, rows: 40 }), isRemoteDesktopViewerOwner: vi.fn().mockReturnValue(false), + getPtyOutputSequence: vi.fn().mockReturnValue(0), ...overrides } as OrcaRuntimeService } @@ -137,7 +140,721 @@ function sendDesktopMultiplexSubscribe( ) } +function sendDesktopSourceRangeSubscribe( + handlers: Map>) => void> +) { + handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) +} + +function sourceRange(start: number, end: number) { + return { + id: 'pty-1', + spanId: `span-${start}-${end}`, + providerGeneration: 5, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: start, + sourceEndSu: end, + displayStart: start, + displayEnd: end, + splittable: true, + transform: { transformed: false, rawLengthSu: end - start, scalarSafe: true } + } as const +} + +async function acknowledgeSourceRangeOverflow( + harness: ReturnType, + dataListener: (data: string, meta?: RuntimeTerminalDataMeta) => void, + data: string, + acknowledge: 'all' | 'first' = 'all' +) { + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + const subscribed = harness.messages + .map((message) => JSON.parse(message).result) + .find((event) => event?.type === 'subscribed') + harness.binaryFrames.splice(0) + dataListener(data, { + seq: data.length, + rawLength: data.length, + sourceRanges: [sourceRange(0, data.length)] + }) + const outputFrames = harness.binaryFrames + .map(decodeTerminalStreamFrame) + .filter((frame) => frame?.opcode === TerminalStreamOpcode.Output) + const acceptedEndByte = outputFrames.reduce( + (total, frame) => total + (frame?.payload.byteLength ?? 0), + 0 + ) + const ackedEndByte = + acknowledge === 'first' ? (outputFrames[0]?.payload.byteLength ?? 0) : acceptedEndByte + expect(ackedEndByte).toBeGreaterThan(0) + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Ack, + streamId: 7, + seq: 2, + payload: encodeTerminalStreamJson({ + streamGeneration: subscribed.streamGeneration, + ackedEndByte + }) + }) + )! + ) + return { acceptedEndByte, ackedEndByte, streamGeneration: subscribed.streamGeneration } +} + +type OverflowRecoverySnapshot = { + data: string + cols: number + rows: number + source?: 'headless' | 'renderer' + seq?: number +} + +function startSourceRangeOverflowHarness(options: { + recover: () => Promise + commit?: () => boolean + onFrame?: (frame: NonNullable>) => boolean | void +}) { + let dataListener: ((data: string, meta?: RuntimeTerminalDataMeta) => void) | undefined + const lifecycle: string[] = [] + const reserve = vi.fn((identity, requiredSeq: number, reason: string) => { + if (reason !== 'ack-pending-overflow') { + return null + } + lifecycle.push('reserve') + return Object.freeze({ + reservationId: 'overflow-replacement', + identity: Object.freeze({ ...identity }), + requiredSeq + }) + }) + const commit = vi.fn(() => { + lifecycle.push('commit') + return options.commit?.() ?? true + }) + const rollback = vi.fn(() => { + lifecycle.push('rollback') + return true + }) + const cancel = vi.fn(() => { + lifecycle.push('cancel') + }) + const harness = startDesktopMultiplexSubscribe( + { + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + settleRemoteTerminalSourceRanges: vi.fn(), + reserveRemoteTerminalSourceRangeReplacement: reserve, + commitRemoteTerminalSourceRangeReplacement: commit, + rollbackRemoteTerminalSourceRangeReplacement: rollback, + cancelRemoteTerminalSourceRanges: cancel, + serializeTerminalBuffer: vi + .fn() + .mockResolvedValueOnce({ + data: 'initial snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: 0 + }) + .mockImplementationOnce(options.recover), + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }) + }, + undefined, + (bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + return frame ? options.onFrame?.(frame) : undefined + } + ) + return { + ...harness, + lifecycle, + reserve, + commit, + rollback, + cancel, + getDataListener: () => dataListener + } +} + describe('terminal multiplex RPC', () => { + it.each(['headless', 'renderer'] as const)( + 'commits a source-range replacement only after the %s snapshot publishes', + async (source) => { + const reservation = { + reservationId: 'replacement-1', + identity: { + ptyId: 'pty-1', + consumerId: 'multiplex:conn-desktop-first-paint:7', + streamGeneration: 'generation' + }, + requiredSeq: 4 + } + const reserve = vi.fn(() => reservation) + const commit = vi.fn(() => true) + const rollback = vi.fn(() => true) + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + reserveRemoteTerminalSourceRangeReplacement: reserve, + commitRemoteTerminalSourceRangeReplacement: commit, + rollbackRemoteTerminalSourceRangeReplacement: rollback, + cancelRemoteTerminalSourceRanges: vi.fn(), + serializeTerminalBuffer: vi + .fn() + .mockResolvedValue({ data: 'snapshot', cols: 120, rows: 40, source, seq: 4 }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + + await vi.waitFor(() => expect(commit).toHaveBeenCalledOnce()) + expect(reserve).toHaveBeenCalledWith(expect.any(Object), 4, 'initial-snapshot') + expect(commit).toHaveBeenCalledWith(reservation, { source, seq: 4 }) + expect(rollback).not.toHaveBeenCalled() + const snapshotEndIndex = harness.binaryFrames.findIndex( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.SnapshotEnd + ) + expect(snapshotEndIndex).toBeGreaterThanOrEqual(0) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + } + ) + + it.each([ + { topology: 'headed renderer snapshot semantics', source: 'renderer' as const }, + { topology: 'headless model snapshot semantics', source: 'headless' as const } + ])( + 'admits only snapshot-covered source spans for a $topology and delivers the trailing span live', + async ({ source }) => { + let dataListener: ((data: string, meta?: RuntimeTerminalDataMeta) => void) | undefined + let modelSequence = 0 + let resolveSnapshot: ( + value: Readonly<{ + data: string + cols: number + rows: number + source: 'headless' | 'renderer' + seq: number + }> + ) => void = () => {} + const publishedSourceEnds: number[] = [] + const prepareExit = vi.fn() + const finalizeExit = vi.fn() + const closeProvider = vi.fn() + const intake = new SshPtyOutputIntake({ + getModelSequence: () => modelSequence, + acceptModel: (event, projection) => { + modelSequence = projection.identity.sequenceEnd + dataListener?.(event.data, { + seq: modelSequence, + rawLength: event.rawLength, + sourceRanges: projection.desktopSpan ? [projection.desktopSpan] : undefined + }) + return { sequence: modelSequence, completion: Promise.resolve() } + }, + project: vi.fn(), + prepareExit, + finalizeExit, + closeProvider, + publishSourceAck: (_providerGeneration, batch, onSettled) => { + publishedSourceEnds.push( + ...batch.acknowledgements.map((acknowledgement) => acknowledgement.creditedEndSu) + ) + onSettled({ ok: true }) + } + }) + const hooks = intake.getRemoteSourceRangeConsumerHooks() + const reserveReplacement = vi.fn(hooks.reserveReplacement) + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: hooks.attach, + settleRemoteTerminalSourceRanges: hooks.settle, + reserveRemoteTerminalSourceRangeReplacement: reserveReplacement, + commitRemoteTerminalSourceRangeReplacement: hooks.commitReplacement, + rollbackRemoteTerminalSourceRangeReplacement: hooks.rollbackReplacement, + cancelRemoteTerminalSourceRanges: hooks.cancel, + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }), + serializeTerminalBuffer: vi.fn( + () => + new Promise<{ + data: string + cols: number + rows: number + source: 'headless' | 'renderer' + seq: number + }>((resolve) => { + resolveSnapshot = resolve + }) + ) + }) + const sourceEvent = ( + spanId: string, + data: string, + sourceStartSu: number + ): SshPtyOutputDataEvent => ({ + id: 'pty-1', + data, + providerGeneration: 9, + ptyIncarnation: 'incarnation-1', + rawLength: data.length, + transformed: false, + source: { + spanId, + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-1', + sourceStartSu, + sourceEndSu: sourceStartSu + data.length + } + }) + const settleDesktop = async (event: SshPtyOutputDataEvent): Promise => { + const receipt = await intake.acceptData(event) + const projectionId = receipt.projection.identity.projectionSemanticsId + intake.publishProjectionPrefix([projectionId], event.data.length, event.rawLength) + expect(intake.settleProjectionPrefix(event.id, event.rawLength)).toBe(event.rawLength) + } + + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + await vi.waitFor(() => expect(dataListener).toBeDefined()) + await settleDesktop(sourceEvent('span-covered', 'snap', 0)) + await settleDesktop(sourceEvent('span-trailing', 'live', 4)) + expect(reserveReplacement).not.toHaveBeenCalled() + + resolveSnapshot({ data: 'snap', cols: 120, rows: 40, source, seq: 4 }) + await vi.waitFor(() => expect(reserveReplacement).toHaveBeenCalledOnce()) + expect(reserveReplacement).toHaveBeenCalledWith(expect.any(Object), 4, 'initial-snapshot') + await vi.waitFor(() => expect(publishedSourceEnds).toEqual([4])) + + const subscribed = harness.messages + .map((message) => JSON.parse(message).result) + .find((event) => event?.type === 'subscribed') + const liveOutput = harness.binaryFrames + .map(decodeTerminalStreamFrame) + .find((frame) => frame?.opcode === TerminalStreamOpcode.Output) + const snapshotOutput = harness.binaryFrames + .map(decodeTerminalStreamFrame) + .filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk) + .map((frame) => decodeTerminalStreamText(frame!.payload)) + .join('') + expect(snapshotOutput).toBe('snap') + expect(liveOutput && decodeTerminalStreamText(liveOutput.payload)).toBe('live') + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Ack, + streamId: 7, + seq: 2, + payload: encodeTerminalStreamJson({ + streamGeneration: subscribed.streamGeneration, + ackedEndByte: liveOutput!.payload.byteLength + }) + }) + )! + ) + await vi.waitFor(() => expect(publishedSourceEnds).toEqual([4, 8])) + await intake.acceptExit({ + id: 'pty-1', + code: 0, + providerGeneration: 9, + ptyIncarnation: 'incarnation-1' + }) + expect(prepareExit).toHaveBeenCalledOnce() + expect(finalizeExit).toHaveBeenCalledOnce() + expect(closeProvider).not.toHaveBeenCalled() + expect(intake.getDebugSnapshot().source).toEqual({ + openedTokens: 0, + ptyIdentities: 0 + }) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + intake.dispose() + } + ) + + it('rolls back replacement admission when snapshot publication is refused', async () => { + const reservation = { + reservationId: 'replacement-1', + identity: { + ptyId: 'pty-1', + consumerId: 'multiplex:conn-desktop-first-paint:7', + streamGeneration: 'generation' + }, + requiredSeq: 4 + } + const commit = vi.fn(() => true) + const rollback = vi.fn(() => true) + const harness = startDesktopMultiplexSubscribe( + { + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + reserveRemoteTerminalSourceRangeReplacement: vi.fn(() => reservation), + commitRemoteTerminalSourceRangeReplacement: commit, + rollbackRemoteTerminalSourceRangeReplacement: rollback, + cancelRemoteTerminalSourceRanges: vi.fn(), + serializeTerminalBuffer: vi.fn().mockResolvedValue({ + data: 'snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: 4 + }) + }, + undefined, + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode !== TerminalStreamOpcode.SnapshotEnd + ) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + + await harness.dispatchPromise + expect(commit).not.toHaveBeenCalled() + expect(rollback).toHaveBeenCalledWith(reservation, 'stream-detached-replacement-aborted') + }) + + it('keeps legacy multiplex clients outside source replacement admission', async () => { + const attach = vi.fn(() => true) + const reserve = vi.fn() + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: attach, + reserveRemoteTerminalSourceRangeReplacement: reserve + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + + expect(attach).not.toHaveBeenCalled() + expect(reserve).not.toHaveBeenCalled() + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + + it('keeps stale parsed source ACKs from releasing in-flight byte credit', async () => { + let dataListener: ((data: string, meta?: RuntimeTerminalDataMeta) => void) | null = null + const settle = vi.fn() + const cancel = vi.fn() + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + settleRemoteTerminalSourceRanges: settle, + reserveRemoteTerminalSourceRangeReplacement: vi.fn(() => null), + commitRemoteTerminalSourceRangeReplacement: vi.fn(() => false), + rollbackRemoteTerminalSourceRangeReplacement: vi.fn(() => false), + cancelRemoteTerminalSourceRanges: cancel, + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + await vi.waitFor(() => expect(dataListener).not.toBeNull()) + await vi.waitFor(() => + expect( + harness.messages + .map((message) => JSON.parse(message).result) + .find((event) => event?.type === 'subscribed') + ).toBeDefined() + ) + const subscribed = harness.messages + .map((message) => JSON.parse(message).result) + .find((event) => event?.type === 'subscribed') + expect(subscribed).toMatchObject({ + capabilities: { ackOutputSourceRanges: 1 }, + streamGeneration: expect.any(String) + }) + + const emitData = dataListener as unknown as ( + data: string, + meta?: RuntimeTerminalDataMeta + ) => void + emitData('ab', { + seq: 2, + rawLength: 2, + sourceRanges: [ + { + id: 'pty-1', + spanId: 'span-1', + providerGeneration: 5, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 2, + displayStart: 0, + displayEnd: 2, + splittable: true, + transform: { transformed: false, rawLengthSu: 2, scalarSafe: true } + } + ] + }) + await vi.waitFor(() => + expect( + harness.binaryFrames.some( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.Output + ) + ).toBe(true) + ) + const output = harness.binaryFrames + .map(decodeTerminalStreamFrame) + .find((frame) => frame?.opcode === TerminalStreamOpcode.Output)! + const acknowledge = (payload: unknown): void => { + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Ack, + streamId: 7, + seq: 2, + payload: encodeTerminalStreamJson(payload) + }) + )! + ) + } + + acknowledge({ + streamGeneration: subscribed.streamGeneration, + ackedEndByte: output.payload.byteLength - 1 + }) + acknowledge({ + streamGeneration: subscribed.streamGeneration, + ackedEndByte: output.payload.byteLength + 1 + }) + acknowledge({ bytes: output.payload.byteLength }) + acknowledge({ epoch: 'notification', watermark: output.payload.byteLength }) + const fillLength = TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES + emitData('x'.repeat(fillLength), { + seq: output.payload.byteLength + fillLength, + rawLength: fillLength, + sourceRanges: [sourceRange(output.payload.byteLength, output.payload.byteLength + fillLength)] + }) + emitData('y'.repeat(64 * 1024), { + seq: output.payload.byteLength + fillLength + 64 * 1024, + rawLength: 64 * 1024, + sourceRanges: [ + sourceRange( + output.payload.byteLength + fillLength, + output.payload.byteLength + fillLength + 64 * 1024 + ) + ] + }) + const outputFramesBeforeStaleAck = harness.binaryFrames.filter( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.Output + ).length + acknowledge({ streamGeneration: 'stale', ackedEndByte: output.payload.byteLength }) + expect(settle).not.toHaveBeenCalled() + expect( + harness.binaryFrames.filter( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.Output + ) + ).toHaveLength(outputFramesBeforeStaleAck) + + acknowledge({ + streamGeneration: subscribed.streamGeneration, + ackedEndByte: output.payload.byteLength + }) + expect(settle).toHaveBeenCalledWith( + expect.objectContaining({ streamGeneration: subscribed.streamGeneration }), + [expect.objectContaining({ spanId: 'span-1', sourceStartSu: 0, sourceEndSu: 2 })] + ) + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Unsubscribe, + streamId: 7, + seq: 3, + payload: new Uint8Array() + }) + )! + ) + expect(cancel).toHaveBeenCalledWith( + expect.any(Object), + expect.arrayContaining([expect.objectContaining({ spanId: `span-2-${2 + fillLength}` })]), + 'stream-detached' + ) + }) + + it('keeps a lossless stream attached across partial ACK and source-token rotation', async () => { + let dataListener: ((data: string, meta?: RuntimeTerminalDataMeta) => void) | null = null + const settle = vi.fn() + const cancel = vi.fn() + const harness = startDesktopMultiplexSubscribe({ + attachRemoteTerminalSourceRangeConsumer: vi.fn(() => true), + settleRemoteTerminalSourceRanges: settle, + reserveRemoteTerminalSourceRangeReplacement: vi.fn(() => null), + commitRemoteTerminalSourceRangeReplacement: vi.fn(() => false), + rollbackRemoteTerminalSourceRangeReplacement: vi.fn(() => false), + cancelRemoteTerminalSourceRanges: cancel, + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(dataListener).not.toBeNull()) + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + const subscribed = harness.messages + .map((message) => JSON.parse(message).result) + .find((event) => event?.type === 'subscribed') + const emitData = dataListener as unknown as ( + data: string, + meta?: RuntimeTerminalDataMeta + ) => void + const acknowledge = (ackedEndByte: number): void => { + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Ack, + streamId: 7, + seq: 2, + payload: encodeTerminalStreamJson({ + streamGeneration: subscribed.streamGeneration, + ackedEndByte + }) + }) + )! + ) + } + harness.binaryFrames.splice(0) + + emitData('a'.repeat(100), { + seq: 100, + rawLength: 100, + sourceRanges: [sourceRange(0, 100)] + }) + await vi.waitFor(() => + expect( + harness.binaryFrames.filter( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.Output + ) + ).toHaveLength(1) + ) + acknowledge(40) + expect(settle).not.toHaveBeenCalled() + + emitData('next', { + seq: 104, + rawLength: 4, + sourceRanges: [ + { + ...sourceRange(100, 104), + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-2' + } + ] + }) + await vi.waitFor(() => + expect( + harness.binaryFrames.filter( + (bytes) => decodeTerminalStreamFrame(bytes)?.opcode === TerminalStreamOpcode.Output + ) + ).toHaveLength(2) + ) + expect(cancel).not.toHaveBeenCalled() + expect(harness.handlers.has(7)).toBe(true) + + acknowledge(100) + expect(settle).toHaveBeenLastCalledWith(expect.any(Object), [ + expect.objectContaining({ deliveryToken: 'token-1', sourceEndSu: 100 }) + ]) + acknowledge(104) + expect(settle).toHaveBeenLastCalledWith(expect.any(Object), [ + expect.objectContaining({ deliveryToken: 'token-2', sourceStartSu: 100 }) + ]) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it.each(['refuses', 'throws'] as const)( 'closes without reserving ACK debt when the transport %s an output frame', async (failureMode) => { @@ -1498,6 +2215,282 @@ describe('terminal multiplex RPC', () => { await dispatchPromise }) + it('replaces UTF-8-expanded overflow ranges before publishing the trailing live range', async () => { + let resolveRecovery: (snapshot: OverflowRecoverySnapshot) => void = () => {} + const harness = startSourceRangeOverflowHarness({ + recover: () => + new Promise((resolve) => { + resolveRecovery = resolve + }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(harness.getDataListener()).toBeDefined()) + + const flooded = '界'.repeat(1024 * 1024) + const credit = await acknowledgeSourceRangeOverflow( + harness, + harness.getDataListener()!, + flooded, + 'first' + ) + await vi.waitFor(() => expect(harness.runtime.serializeTerminalBuffer).toHaveBeenCalledTimes(2)) + const trailing = 'trailing-live' + harness.getDataListener()!(trailing, { + seq: flooded.length + trailing.length, + rawLength: trailing.length, + sourceRanges: [sourceRange(flooded.length, flooded.length + trailing.length)] + }) + resolveRecovery({ + data: 'authoritative snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: flooded.length + }) + + await vi.waitFor(() => expect(harness.commit).toHaveBeenCalledOnce()) + expect(harness.reserve).toHaveBeenLastCalledWith( + expect.objectContaining({ streamGeneration: expect.any(String) }), + flooded.length, + 'ack-pending-overflow' + ) + expect(harness.lifecycle).toEqual(['reserve', 'commit']) + const recoveryFrames = harness.binaryFrames.map(decodeTerminalStreamFrame) + const snapshotEnd = recoveryFrames.findLastIndex( + (frame) => frame?.opcode === TerminalStreamOpcode.SnapshotEnd + ) + const trailingOutput = recoveryFrames.findIndex( + (frame, index) => + index > snapshotEnd && + frame?.opcode === TerminalStreamOpcode.Output && + decodeTerminalStreamText(frame.payload) === trailing + ) + expect(snapshotEnd).toBeGreaterThanOrEqual(0) + expect(trailingOutput).toBeGreaterThan(snapshotEnd) + expect(harness.rollback).not.toHaveBeenCalled() + const trailingBytes = recoveryFrames[trailingOutput]!.payload.byteLength + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Ack, + streamId: 7, + seq: 3, + payload: encodeTerminalStreamJson({ + streamGeneration: credit.streamGeneration, + ackedEndByte: credit.acceptedEndByte + trailingBytes + }) + }) + )! + ) + expect(harness.runtime.settleRemoteTerminalSourceRanges).toHaveBeenLastCalledWith( + expect.any(Object), + [sourceRange(flooded.length, flooded.length + trailing.length)] + ) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + + it('rolls back overflow replacement before detaching on partial frame publication', async () => { + let recoveryStarted = false + const harness = startSourceRangeOverflowHarness({ + recover: async () => ({ + data: 'authoritative snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: 3 * 1024 * 1024 + }), + onFrame: (frame) => { + if (frame.opcode === TerminalStreamOpcode.SnapshotStart) { + recoveryStarted = + decodeTerminalStreamJson<{ reason?: string }>(frame.payload)?.reason === + 'ack-pending-overflow' + } + if (recoveryStarted && frame.opcode === TerminalStreamOpcode.SnapshotChunk) { + return false + } + return undefined + } + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(harness.getDataListener()).toBeDefined()) + + const flooded = 'x'.repeat(3 * 1024 * 1024) + await acknowledgeSourceRangeOverflow(harness, harness.getDataListener()!, flooded) + await harness.dispatchPromise + + expect(harness.commit).not.toHaveBeenCalled() + expect(harness.rollback).toHaveBeenCalledOnce() + expect(harness.lifecycle).toEqual(['reserve', 'rollback', 'cancel']) + }) + + it('rolls back overflow replacement before generic detach when commit rejects', async () => { + const flooded = 'x'.repeat(3 * 1024 * 1024) + const harness = startSourceRangeOverflowHarness({ + recover: async () => ({ + data: 'authoritative snapshot', + cols: 120, + rows: 40, + source: 'renderer', + seq: flooded.length + }), + commit: () => false + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(harness.getDataListener()).toBeDefined()) + + await acknowledgeSourceRangeOverflow(harness, harness.getDataListener()!, flooded) + await vi.waitFor(() => expect(harness.cancel).toHaveBeenCalledOnce()) + + expect(harness.commit).toHaveBeenCalledOnce() + expect(harness.rollback).toHaveBeenCalledOnce() + expect(harness.lifecycle).toEqual(['reserve', 'commit', 'rollback', 'cancel']) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + + it('rolls back a held overflow replacement when its stream detaches', async () => { + const flooded = 'x'.repeat(3 * 1024 * 1024) + let harness: ReturnType + harness = startSourceRangeOverflowHarness({ + recover: async () => ({ + data: 'authoritative snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: flooded.length + }), + onFrame: (frame) => { + if ( + frame.opcode !== TerminalStreamOpcode.SnapshotStart || + decodeTerminalStreamJson<{ reason?: string }>(frame.payload)?.reason !== + 'ack-pending-overflow' + ) { + return + } + harness.handlers.get(7)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Unsubscribe, + streamId: 7, + seq: 3, + payload: new Uint8Array() + }) + )! + ) + } + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(harness.getDataListener()).toBeDefined()) + + await acknowledgeSourceRangeOverflow(harness, harness.getDataListener()!, flooded) + await vi.waitFor(() => expect(harness.handlers.has(7)).toBe(false)) + + expect(harness.commit).not.toHaveBeenCalled() + expect(harness.rollback).toHaveBeenCalledOnce() + expect(harness.lifecycle).toEqual(['reserve', 'rollback', 'cancel']) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + + it('rolls back overflow replacement before a same-slot generation succeeds it', async () => { + const flooded = 'x'.repeat(3 * 1024 * 1024) + let harness: ReturnType + harness = startSourceRangeOverflowHarness({ + recover: async () => ({ + data: 'authoritative snapshot', + cols: 120, + rows: 40, + source: 'headless', + seq: flooded.length + }), + onFrame: (frame) => { + if ( + frame.opcode !== TerminalStreamOpcode.SnapshotStart || + decodeTerminalStreamJson<{ reason?: string }>(frame.payload)?.reason !== + 'ack-pending-overflow' + ) { + return + } + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 3, + payload: encodeTerminalStreamJson({ + streamId: 7, + terminal: 'terminal-1', + client: { id: 'desktop-2', type: 'desktop' }, + capabilities: { ackOutput: 1, ackOutputSourceRanges: 1 } + }) + }) + )! + ) + } + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(harness.getDataListener()).toBeDefined()) + + await acknowledgeSourceRangeOverflow(harness, harness.getDataListener()!, flooded) + await vi.waitFor(() => + expect( + harness.messages.filter((message) => JSON.parse(message).result?.type === 'subscribed') + .length + ).toBe(2) + ) + + expect(harness.commit).not.toHaveBeenCalled() + expect(harness.rollback).toHaveBeenCalledOnce() + expect(harness.lifecycle.slice(0, 3)).toEqual(['reserve', 'rollback', 'cancel']) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + + it('does not publish or trim overflow output without serialized source identity', async () => { + const flooded = 'x'.repeat(3 * 1024 * 1024) + const harness = startSourceRangeOverflowHarness({ + recover: async () => ({ + data: 'unattributed snapshot', + cols: 120, + rows: 40, + seq: flooded.length + }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopSourceRangeSubscribe(harness.handlers) + await vi.waitFor(() => expect(harness.getDataListener()).toBeDefined()) + const frameCount = harness.binaryFrames.length + + await acknowledgeSourceRangeOverflow(harness, harness.getDataListener()!, flooded) + await vi.waitFor(() => expect(harness.cancel).toHaveBeenCalledOnce()) + + expect(harness.reserve.mock.calls.some((call) => call[2] === 'ack-pending-overflow')).toBe( + false + ) + expect(harness.commit).not.toHaveBeenCalled() + expect(harness.rollback).not.toHaveBeenCalled() + expect( + harness.binaryFrames + .slice(frameCount) + .map(decodeTerminalStreamFrame) + .some( + (frame) => + frame?.opcode === TerminalStreamOpcode.SnapshotStart && + decodeTerminalStreamJson<{ reason?: string }>(frame.payload)?.reason === + 'ack-pending-overflow' + ) + ).toBe(false) + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it('caps stalled ACK output and snapshots before resuming retained tail frames', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] diff --git a/src/main/runtime/rpc/terminal-output-frame-chunks-equivalence.test.ts b/src/main/runtime/rpc/terminal-output-frame-chunks-equivalence.test.ts index 3de93dcfe0f..233aa3e4bc2 100644 --- a/src/main/runtime/rpc/terminal-output-frame-chunks-equivalence.test.ts +++ b/src/main/runtime/rpc/terminal-output-frame-chunks-equivalence.test.ts @@ -47,12 +47,13 @@ function* legacyIterateTerminalOutputFrameChunks( yield { opcode: TerminalStreamOpcode.OutputSpan, bytes: encodeTerminalStreamJson({ data, rawLength, transformed: true }), + displayLength: data.length, seq: meta?.seq } return } if (!legacyByteLengthExceeds(data, TERMINAL_STREAM_CHUNK_BYTES)) { - yield { bytes: encodeTerminalStreamText(data), seq: meta?.seq } + yield { bytes: encodeTerminalStreamText(data), displayLength: data.length, seq: meta?.seq } return } const canPreserveChunkSeq = typeof meta?.seq === 'number' && rawLength === data.length @@ -83,11 +84,18 @@ function* legacyIterateTerminalOutputFrameChunks( if (nextChunk) { if (shouldDelayFinalSeq) { if (delayedChunk) { - yield { bytes: encodeTerminalStreamText(delayedChunk.text) } + yield { + bytes: encodeTerminalStreamText(delayedChunk.text), + displayLength: delayedChunk.text.length + } } delayedChunk = nextChunk } else { - yield { bytes: encodeTerminalStreamText(nextChunk.text), seq: nextChunk.seq } + yield { + bytes: encodeTerminalStreamText(nextChunk.text), + displayLength: nextChunk.text.length, + seq: nextChunk.seq + } } } } @@ -99,17 +107,28 @@ function* legacyIterateTerminalOutputFrameChunks( if (shouldDelayFinalSeq) { if (finalChunk) { if (delayedChunk) { - yield { bytes: encodeTerminalStreamText(delayedChunk.text) } + yield { + bytes: encodeTerminalStreamText(delayedChunk.text), + displayLength: delayedChunk.text.length + } } delayedChunk = finalChunk } if (delayedChunk) { - yield { bytes: encodeTerminalStreamText(delayedChunk.text), seq: meta.seq } + yield { + bytes: encodeTerminalStreamText(delayedChunk.text), + displayLength: delayedChunk.text.length, + seq: meta.seq + } } return } if (finalChunk) { - yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: finalChunk.seq } + yield { + bytes: encodeTerminalStreamText(finalChunk.text), + displayLength: finalChunk.text.length, + seq: finalChunk.seq + } } } diff --git a/src/main/runtime/rpc/terminal-output-frame-chunks.ts b/src/main/runtime/rpc/terminal-output-frame-chunks.ts index 167b6c20494..91bbc506680 100644 --- a/src/main/runtime/rpc/terminal-output-frame-chunks.ts +++ b/src/main/runtime/rpc/terminal-output-frame-chunks.ts @@ -4,6 +4,7 @@ import { encodeTerminalStreamText } from '../../../shared/terminal-stream-protocol' import { TERMINAL_STREAM_CHUNK_BYTES } from '../../../shared/terminal-multiplex-flow-control' +import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range' import { terminalStreamByteLength } from './terminal-stream-byte-length' export type TerminalOutputMeta = { @@ -11,12 +12,15 @@ export type TerminalOutputMeta = { rawLength?: number transformed?: boolean cwd?: string + sourceRanges?: readonly TerminalOutputSourceRange[] } export type TerminalOutputFrameChunk = { bytes: Uint8Array + displayLength: number seq?: number opcode?: TerminalStreamOpcode + sourceRanges?: readonly TerminalOutputSourceRange[] } export const TERMINAL_STREAM_BYTE_PROBE_CODE_UNITS = 8 * 1024 @@ -55,12 +59,19 @@ export function* iterateTerminalOutputFrameChunks( yield { opcode: TerminalStreamOpcode.OutputSpan, bytes: encodeTerminalStreamJson({ data, rawLength, transformed: true }), - seq: meta?.seq + displayLength: data.length, + seq: meta?.seq, + sourceRanges: meta?.sourceRanges } return } if (!exceedsTerminalStreamChunkBytes(data)) { - yield { bytes: encodeTerminalStreamText(data), seq: meta?.seq } + yield { + bytes: encodeTerminalStreamText(data), + displayLength: data.length, + seq: meta?.seq, + sourceRanges: meta?.sourceRanges + } return } const canPreserveChunkSeq = typeof meta?.seq === 'number' && rawLength === data.length @@ -109,11 +120,23 @@ export function* iterateTerminalOutputFrameChunks( const nextChunk = takeChunk(index) if (shouldDelayFinalSeq) { if (delayedChunk) { - yield { bytes: encodeTerminalStreamText(delayedChunk.text) } + yield { + bytes: encodeTerminalStreamText(delayedChunk.text), + displayLength: delayedChunk.text.length + } } delayedChunk = nextChunk } else { - yield { bytes: encodeTerminalStreamText(nextChunk.text), seq: nextChunk.seq } + yield { + bytes: encodeTerminalStreamText(nextChunk.text), + displayLength: nextChunk.text.length, + seq: nextChunk.seq, + sourceRanges: sliceTerminalOutputSourceRanges( + meta?.sourceRanges, + chunkStart - nextChunk.text.length, + chunkStart + ) + } } } chunkBytes += partBytes @@ -124,10 +147,65 @@ export function* iterateTerminalOutputFrameChunks( if (shouldDelayFinalSeq) { // Why: only the final frame can safely carry the high-water mark when rawLength can't map back to UTF-16 offsets. if (delayedChunk) { - yield { bytes: encodeTerminalStreamText(delayedChunk.text) } + yield { + bytes: encodeTerminalStreamText(delayedChunk.text), + displayLength: delayedChunk.text.length + } + } + yield { + bytes: encodeTerminalStreamText(finalChunk.text), + displayLength: finalChunk.text.length, + seq: meta.seq } - yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: meta.seq } return } - yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: finalChunk.seq } + yield { + bytes: encodeTerminalStreamText(finalChunk.text), + displayLength: finalChunk.text.length, + seq: finalChunk.seq, + sourceRanges: sliceTerminalOutputSourceRanges( + meta?.sourceRanges, + data.length - finalChunk.text.length, + data.length + ) + } +} + +export function sliceTerminalOutputSourceRanges( + sourceRanges: readonly TerminalOutputSourceRange[] | undefined, + displayStartOffset: number, + displayEndOffset: number +): readonly TerminalOutputSourceRange[] | undefined { + if (!sourceRanges || sourceRanges.length === 0) { + return undefined + } + const baseDisplayStart = sourceRanges[0]!.displayStart + const sliceStart = baseDisplayStart + displayStartOffset + const sliceEnd = baseDisplayStart + displayEndOffset + const selected: TerminalOutputSourceRange[] = [] + for (const range of sourceRanges) { + const start = Math.max(sliceStart, range.displayStart) + const end = Math.min(sliceEnd, range.displayEnd) + if (end <= start) { + continue + } + if (!range.splittable && (start !== range.displayStart || end !== range.displayEnd)) { + throw new Error('terminal_source_range_indivisible_split') + } + const sourceStartSu = range.sourceStartSu + (start - range.displayStart) + selected.push( + Object.freeze({ + ...range, + displayStart: start, + displayEnd: end, + sourceStartSu, + sourceEndSu: sourceStartSu + (end - start), + transform: Object.freeze({ + ...range.transform, + rawLengthSu: end - start + }) + }) + ) + } + return Object.freeze(selected) } diff --git a/src/main/runtime/rpc/terminal-output-frame-source-ranges.test.ts b/src/main/runtime/rpc/terminal-output-frame-source-ranges.test.ts new file mode 100644 index 00000000000..0a27718f71c --- /dev/null +++ b/src/main/runtime/rpc/terminal-output-frame-source-ranges.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest' +import { TERMINAL_STREAM_CHUNK_BYTES } from '../../../shared/terminal-multiplex-flow-control' +import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range' +import { iterateTerminalOutputFrameChunks } from './terminal-output-frame-chunks' + +function range(overrides: Partial): TerminalOutputSourceRange { + return { + id: 'pty-1', + spanId: 'span-1', + providerGeneration: 2, + clientGeneration: 3, + ownerGeneration: 4, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 1, + displayStart: 0, + displayEnd: 1, + splittable: true, + transform: { transformed: false, rawLengthSu: 1, scalarSafe: true }, + ...overrides + } +} + +describe('terminal output frame source ranges', () => { + it('maps encoded chunk boundaries to exact ordered source subranges', () => { + const data = 'a'.repeat(TERMINAL_STREAM_CHUNK_BYTES + 2) + const frames = Array.from( + iterateTerminalOutputFrameChunks(data, { + seq: data.length, + rawLength: data.length, + sourceRanges: [ + range({ + sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES - 1, + displayEnd: TERMINAL_STREAM_CHUNK_BYTES - 1, + transform: { + transformed: false, + rawLengthSu: TERMINAL_STREAM_CHUNK_BYTES - 1, + scalarSafe: true + } + }), + range({ + spanId: 'span-2', + sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES - 1, + sourceEndSu: data.length, + displayStart: TERMINAL_STREAM_CHUNK_BYTES - 1, + displayEnd: data.length, + transform: { + transformed: false, + rawLengthSu: data.length - TERMINAL_STREAM_CHUNK_BYTES + 1, + scalarSafe: true + } + }) + ] + }) + ) + + expect(frames).toHaveLength(2) + expect(frames[0]?.sourceRanges).toEqual([ + expect.objectContaining({ + spanId: 'span-1', + sourceStartSu: 0, + sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES - 1 + }), + expect.objectContaining({ + spanId: 'span-2', + sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES - 1, + sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES + }) + ]) + expect(frames[1]?.sourceRanges).toEqual([ + expect.objectContaining({ + spanId: 'span-2', + sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES, + sourceEndSu: data.length + }) + ]) + }) + + it('keeps transformed source ranges indivisible', () => { + const sourceRange = range({ + sourceEndSu: 9, + displayEnd: 3, + splittable: false, + transform: { transformed: true, rawLengthSu: 9, scalarSafe: false } + }) + const [frame] = iterateTerminalOutputFrameChunks('xyz', { + seq: 9, + rawLength: 9, + transformed: true, + sourceRanges: [sourceRange] + }) + + expect(frame?.sourceRanges).toEqual([sourceRange]) + }) +}) diff --git a/src/main/runtime/rpc/terminal-source-range-ledger.test.ts b/src/main/runtime/rpc/terminal-source-range-ledger.test.ts new file mode 100644 index 00000000000..ee07403ca58 --- /dev/null +++ b/src/main/runtime/rpc/terminal-source-range-ledger.test.ts @@ -0,0 +1,332 @@ +import { describe, expect, it, vi } from 'vitest' +import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range' +import { + TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES, + TerminalSourceRangeLedger +} from './terminal-source-range-ledger' +import { TerminalSourceRangeRegistry } from './terminal-source-range-registry' + +function range(overrides: Partial = {}): TerminalOutputSourceRange { + return { + id: 'pty-1', + spanId: 'span-1', + providerGeneration: 4, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4, + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }, + ...overrides + } +} + +describe('TerminalSourceRangeLedger', () => { + it('advances partial byte credit without settling its covering source frame', () => { + const release = vi.fn() + const ledger = new TerminalSourceRangeLedger('generation-1', { + canReserve: () => true, + reserve: () => true, + release, + close: () => {} + }) + ledger.accept(100, 100, [ + range({ + sourceEndSu: 100, + displayEnd: 100, + transform: { transformed: false, rawLengthSu: 100, scalarSafe: true } + }) + ]) + + expect(ledger.acknowledge('stale-generation', 40)).toEqual({ + status: 'stale-generation', + settled: [] + }) + expect(ledger.acknowledge('generation-1', 40)).toEqual({ + status: 'accepted', + acknowledgedBytes: 40, + settled: [] + }) + expect(ledger.getDebugSnapshot()).toMatchObject({ + ackedEndByte: 40, + retainedBytes: 60, + frames: 1 + }) + expect(release).toHaveBeenLastCalledWith(40) + + expect(ledger.acknowledge('generation-1', 100)).toMatchObject({ + status: 'accepted', + acknowledgedBytes: 60, + settled: [{ spanId: 'span-1', sourceStartSu: 0, sourceEndSu: 100 }] + }) + expect(ledger.getDebugSnapshot()).toMatchObject({ + ackedEndByte: 100, + retainedBytes: 0, + frames: 0 + }) + expect(release.mock.calls).toEqual([[40], [60]]) + }) + + it('admits a contiguous recovered delivery while the prior token remains unsettled', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + expect(ledger.accept(4, 4, [range()])).not.toBeNull() + expect( + ledger.accept(4, 4, [ + range({ + spanId: 'span-recovered', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'token-2', + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + }) + ]) + ).not.toBeNull() + + expect(ledger.acknowledge('generation-1', 4)).toMatchObject({ + status: 'accepted', + settled: [{ spanId: 'span-1', deliveryToken: 'token-1' }] + }) + expect(ledger.acknowledge('generation-1', 8)).toMatchObject({ + status: 'accepted', + settled: [{ spanId: 'span-recovered', deliveryToken: 'token-2' }] + }) + expect( + ledger.prepareAccept(4, 4, [ + range({ + spanId: 'span-stale', + sourceStartSu: 8, + sourceEndSu: 12, + displayStart: 8, + displayEnd: 12 + }) + ]).status + ).toBe('cross-generation') + }) + + it('settles only complete frames when a cumulative ACK crosses frame boundaries', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + ledger.accept(5, 4, [range()]) + ledger.accept(7, 5, [ + range({ + spanId: 'span-2', + sourceStartSu: 4, + sourceEndSu: 9, + displayStart: 4, + displayEnd: 9, + transform: { transformed: false, rawLengthSu: 5, scalarSafe: true } + }) + ]) + + expect(ledger.acknowledge('generation-1', 3)).toEqual({ + status: 'accepted', + acknowledgedBytes: 3, + settled: [] + }) + expect(ledger.getDebugSnapshot()).toMatchObject({ + ackedEndByte: 3, + retainedBytes: 9, + frames: 2 + }) + expect(ledger.acknowledge('generation-1', 12)).toMatchObject({ + status: 'accepted', + acknowledgedBytes: 9, + settled: [{ spanId: 'span-1' }, { spanId: 'span-2' }] + }) + }) + + it('rejects malformed coverage, gaps, and later cross-token mappings', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + expect(ledger.accept(5, 3, [range()])).toBeNull() + expect(ledger.accept(5, 4, [range()])).not.toBeNull() + expect( + ledger.accept(5, 4, [ + range({ + spanId: 'gap', + sourceStartSu: 5, + sourceEndSu: 9, + displayStart: 4, + displayEnd: 8 + }) + ]) + ).toBeNull() + expect( + ledger.prepareAccept(5, 4, [ + range({ + spanId: 'other', + deliveryToken: 'token-2', + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + }) + ]).status + ).toBe('cross-generation') + expect(ledger.getDebugSnapshot()).toMatchObject({ retainedBytes: 5, frames: 1 }) + }) + + it('rejects mixing mapped and unmapped output in either order', () => { + const mapped = new TerminalSourceRangeLedger('mapped') + expect(mapped.accept(5, 4, [range()])).not.toBeNull() + expect(mapped.prepareAccept(1, 1, []).status).toBe('invalid') + + const unmapped = new TerminalSourceRangeLedger('unmapped') + expect(unmapped.accept(1, 1, [])).not.toBeNull() + expect(unmapped.prepareAccept(5, 4, [range()]).status).toBe('invalid') + }) + + it('rejects excessive, stale, notification-shaped, and late settlement', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + ledger.accept(5, 4, [range()]) + + expect(ledger.acknowledge('generation-2', 5).status).toBe('stale-generation') + expect(ledger.acknowledge('generation-1', 6).status).toBe('excessive') + expect(ledger.acknowledge('generation-1', Number.NaN).status).toBe('invalid') + const transfer = ledger.beginTransfer() + transfer.commit() + expect(ledger.acknowledge('generation-1', 5).status).toBe('invalid') + expect(ledger.getDebugSnapshot()).toMatchObject({ ackedEndByte: 0, closed: true }) + }) + + it('restores all mappings when an atomic transfer rolls back', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + ledger.accept(5, 4, [range()]) + const transfer = ledger.beginTransfer() + + expect(transfer.frames).toHaveLength(1) + expect(ledger.canAccept(1)).toBe(false) + transfer.rollback() + expect(ledger.getDebugSnapshot()).toMatchObject({ + retainedBytes: 5, + frames: 1, + transferring: false, + closed: false + }) + + const retry = ledger.beginTransfer() + retry.commit() + expect(ledger.getDebugSnapshot()).toMatchObject({ + retainedBytes: 0, + frames: 0, + closed: true + }) + }) + + it('replaces covered mappings without synthesizing encoded-byte credit', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + ledger.accept(5, 4, [range()], 4) + ledger.accept( + 7, + 4, + [ + range({ + spanId: 'span-trailing', + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + }) + ], + 8 + ) + const replacement = ledger.planSourceRangeReplacement(8) + + expect(replacement).not.toBeNull() + expect(() => replacement?.commit()).not.toThrow() + expect(ledger.acknowledge('generation-1', 12)).toMatchObject({ + status: 'accepted', + acknowledgedBytes: 12, + settled: [] + }) + expect( + ledger.accept( + 3, + 2, + [ + range({ + spanId: 'span-live', + sourceStartSu: 8, + sourceEndSu: 10, + displayStart: 8, + displayEnd: 10, + transform: { transformed: false, rawLengthSu: 2, scalarSafe: true } + }) + ], + 10 + ) + ).not.toBeNull() + }) + + it('rejects an admitted trailing mapping before authoritative commit', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + ledger.accept(5, 4, [range()], 4) + ledger.accept( + 7, + 4, + [ + range({ + spanId: 'span-trailing', + sourceStartSu: 4, + sourceEndSu: 8, + displayStart: 4, + displayEnd: 8 + }) + ], + 8 + ) + + expect(ledger.planSourceRangeReplacement(4)).toBeNull() + }) + + it('rejects an unsequenced mapped frame before replacement commit', () => { + const ledger = new TerminalSourceRangeLedger('generation-1') + ledger.accept(5, 4, [range()]) + + expect(ledger.planSourceRangeReplacement(4)).toBeNull() + }) + + it('rolls back a pre-send admission without accepting a byte boundary', () => { + const registry = new TerminalSourceRangeRegistry() + const ledger = registry.open('generation-1')! + const prepared = ledger.prepareAccept(5, 4, [range()]) + expect(prepared.status).toBe('ready') + if (prepared.status !== 'ready') { + throw new Error('expected source range admission') + } + expect(registry.getDebugSnapshot().retainedBytes).toBe(5) + + prepared.admission.rollback() + + expect(ledger.getDebugSnapshot()).toMatchObject({ + acceptedEndByte: 0, + retainedBytes: 0, + frames: 0 + }) + expect(registry.getDebugSnapshot().retainedBytes).toBe(0) + }) + + it('bounds aggregate retained mapping bytes and releases them on ACK and close', () => { + const registry = new TerminalSourceRangeRegistry() + const ledgers = Array.from({ length: 9 }, (_, index) => registry.open(`stream-${index}`)!) + for (const ledger of ledgers.slice(0, 8)) { + expect(ledger.accept(TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES, 0, [])).not.toBeNull() + } + expect(ledgers[8]!.canAccept(1)).toBe(false) + expect(registry.getDebugSnapshot().retainedBytes).toBe(16 * 1024 * 1024) + + expect(ledgers[0]!.acknowledge('stream-0', TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES).status).toBe( + 'accepted' + ) + expect(ledgers[8]!.accept(1, 0, [])).not.toBeNull() + for (const ledger of ledgers) { + ledger.close() + } + expect(registry.getDebugSnapshot()).toEqual({ streams: 0, retainedBytes: 0 }) + }) +}) diff --git a/src/main/runtime/rpc/terminal-source-range-ledger.ts b/src/main/runtime/rpc/terminal-source-range-ledger.ts new file mode 100644 index 00000000000..de032e077bd --- /dev/null +++ b/src/main/runtime/rpc/terminal-source-range-ledger.ts @@ -0,0 +1,310 @@ +import { + sameTerminalOutputSourceIdentity, + type TerminalOutputSourceRange +} from '../../../shared/terminal-output-source-range' +import { + canPlanTerminalSourceRangeReplacement, + freezeTerminalOutputSourceRanges, + replaceTerminalSourceRangeFrames, + type TerminalSourceRangeFrame, + validateTerminalSourceRangeFrame +} from './terminal-source-range-validation' + +export type { TerminalSourceRangeFrame } from './terminal-source-range-validation' + +export const TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES = 2 * 1024 * 1024 + +export type TerminalSourceRangeAckResult = + | { + status: 'accepted' + acknowledgedBytes: number + settled: readonly TerminalOutputSourceRange[] + } + | { status: 'duplicate'; settled: readonly [] } + | { + status: 'invalid' | 'stale-generation' | 'excessive' | 'cross-generation' + settled: readonly [] + } + +export type TerminalSourceRangeAdmission = Readonly<{ + frame: TerminalSourceRangeFrame + commit: () => boolean + rollback: () => void +}> + +export type TerminalSourceRangeAdmissionResult = + | { status: 'ready'; admission: TerminalSourceRangeAdmission } + | { status: 'capacity' | 'invalid' | 'cross-generation' } + +export type TerminalSourceRangeTransfer = Readonly<{ + frames: readonly TerminalSourceRangeFrame[] + commit: () => void + rollback: () => void +}> + +export type TerminalSourceRangeBudget = { + canReserve: (bytes: number) => boolean + reserve: (bytes: number) => boolean + release: (bytes: number) => void + close: () => void +} + +const STANDALONE_BUDGET: TerminalSourceRangeBudget = { + canReserve: () => true, + reserve: () => true, + release: () => {}, + close: () => {} +} + +function isRecoveredSourceIdentity( + previous: TerminalOutputSourceRange, + next: TerminalOutputSourceRange +): boolean { + return ( + previous.id === next.id && + previous.providerGeneration === next.providerGeneration && + previous.ptyIncarnation === next.ptyIncarnation && + next.clientGeneration > previous.clientGeneration && + next.ownerGeneration > previous.ownerGeneration && + next.deliveryToken !== previous.deliveryToken + ) +} + +export class TerminalSourceRangeLedger { + private acceptedEndByte = 0 + private ackedEndByte = 0 + private retainedBytes = 0 + private frames: TerminalSourceRangeFrame[] = [] + private mappingMode: 'mapped' | 'unmapped' | null = null + private boundRange: TerminalOutputSourceRange | null = null + private mappedSourceEndSu: number | null = null + private mappedDisplayEnd: number | null = null + private pending: TerminalSourceRangeAdmission | null = null + private transferring = false + private closed = false + + constructor( + readonly streamGeneration: string, + private readonly budget: TerminalSourceRangeBudget = STANDALONE_BUDGET + ) { + if (!streamGeneration) { + throw new Error('terminal_source_range_generation_required') + } + } + + canAccept(encodedBytes: number): boolean { + return ( + !this.closed && + !this.transferring && + !this.pending && + Number.isSafeInteger(encodedBytes) && + encodedBytes > 0 && + this.retainedBytes + encodedBytes <= TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES && + this.budget.canReserve(encodedBytes) + ) + } + + prepareAccept( + encodedBytes: number, + displayLength: number, + sourceRanges: readonly TerminalOutputSourceRange[], + outputSeq?: number + ): TerminalSourceRangeAdmissionResult { + if (!this.canAccept(encodedBytes)) { + return { status: 'capacity' } + } + if (!validateTerminalSourceRangeFrame(displayLength, sourceRanges)) { + return { status: 'invalid' } + } + const nextMappingMode = sourceRanges.length > 0 ? 'mapped' : 'unmapped' + if (this.mappingMode && this.mappingMode !== nextMappingMode) { + return { status: 'invalid' } + } + const first = sourceRanges[0] + if ( + first && + (this.boundRange + ? !sameTerminalOutputSourceIdentity(this.boundRange, first) && + !isRecoveredSourceIdentity(this.boundRange, first) + : !sourceRanges.every((range) => sameTerminalOutputSourceIdentity(first, range))) + ) { + return { status: 'cross-generation' } + } + if ( + first && + ((this.mappedSourceEndSu !== null && first.sourceStartSu !== this.mappedSourceEndSu) || + (this.mappedDisplayEnd !== null && first.displayStart !== this.mappedDisplayEnd)) + ) { + return { status: 'invalid' } + } + if (!this.budget.reserve(encodedBytes)) { + return { status: 'capacity' } + } + const ranges = freezeTerminalOutputSourceRanges(sourceRanges) + const frame = Object.freeze({ + encodedStartByte: this.acceptedEndByte, + encodedEndByte: this.acceptedEndByte + encodedBytes, + displayLength, + ...(typeof outputSeq === 'number' ? { outputSeq } : {}), + sourceRanges: ranges + }) + let finished = false + const admission: TerminalSourceRangeAdmission = Object.freeze({ + frame, + commit: () => { + if (finished || this.pending !== admission || this.closed || this.transferring) { + return false + } + finished = true + this.pending = null + this.acceptedEndByte = frame.encodedEndByte + this.retainedBytes += encodedBytes + this.frames.push(frame) + this.mappingMode ??= nextMappingMode + const last = ranges.at(-1) + if (first && last) { + this.boundRange = first + this.mappedSourceEndSu = last.sourceEndSu + this.mappedDisplayEnd = last.displayEnd + } + return true + }, + rollback: () => { + if (finished) { + return + } + finished = true + if (this.pending === admission) { + this.pending = null + } + this.budget.release(encodedBytes) + } + }) + this.pending = admission + return { status: 'ready', admission } + } + + accept( + encodedBytes: number, + displayLength: number, + sourceRanges: readonly TerminalOutputSourceRange[], + outputSeq?: number + ): TerminalSourceRangeFrame | null { + const prepared = this.prepareAccept(encodedBytes, displayLength, sourceRanges, outputSeq) + if (prepared.status !== 'ready' || !prepared.admission.commit()) { + return null + } + return prepared.admission.frame + } + + acknowledge(streamGeneration: string, ackedEndByte: number): TerminalSourceRangeAckResult { + if (streamGeneration !== this.streamGeneration) { + return { status: 'stale-generation', settled: [] } + } + if (this.closed || this.transferring || this.pending) { + return { status: 'invalid', settled: [] } + } + if ( + !Number.isSafeInteger(ackedEndByte) || + ackedEndByte < 0 || + ackedEndByte < this.ackedEndByte + ) { + return { status: 'invalid', settled: [] } + } + if (ackedEndByte > this.acceptedEndByte) { + return { status: 'excessive', settled: [] } + } + if (ackedEndByte === this.ackedEndByte) { + return { status: 'duplicate', settled: [] } + } + const acknowledgedBytes = ackedEndByte - this.ackedEndByte + const settled: TerminalOutputSourceRange[] = [] + let frameCount = 0 + for (const frame of this.frames) { + if (frame.encodedEndByte > ackedEndByte) { + break + } + settled.push(...frame.sourceRanges) + frameCount++ + } + this.ackedEndByte = ackedEndByte + this.frames.splice(0, frameCount) + this.retainedBytes -= acknowledgedBytes + this.budget.release(acknowledgedBytes) + return { status: 'accepted', acknowledgedBytes, settled: Object.freeze(settled) } + } + + planSourceRangeReplacement(snapshotSeq: number): Readonly<{ commit: () => void }> | null { + const unavailable = this.closed || this.transferring || this.pending + if (unavailable || !canPlanTerminalSourceRangeReplacement(this.frames, snapshotSeq)) { + return null + } + const replacement = replaceTerminalSourceRangeFrames(this.frames, snapshotSeq) + let committed = false + return Object.freeze({ + commit: () => { + if (committed || this.closed) { + return + } + committed = true + Object.assign(this, replacement) + } + }) + } + + beginTransfer(): TerminalSourceRangeTransfer { + if (this.closed || this.transferring || this.pending) { + throw new Error('terminal_source_range_transfer_invalid') + } + this.transferring = true + let finished = false + const frames = Object.freeze(this.frames.slice()) + return Object.freeze({ + frames, + commit: () => { + if (finished) { + return + } + finished = true + this.budget.release(this.retainedBytes) + this.frames = [] + this.retainedBytes = 0 + this.transferring = false + this.closed = true + this.budget.close() + }, + rollback: () => { + if (finished) { + return + } + finished = true + this.transferring = false + } + }) + } + + close(): void { + if (this.closed) { + return + } + this.pending?.rollback() + this.budget.release(this.retainedBytes) + this.frames = [] + this.retainedBytes = 0 + this.transferring = false + this.closed = true + this.budget.close() + } + + getDebugSnapshot() { + return { + acceptedEndByte: this.acceptedEndByte, + ackedEndByte: this.ackedEndByte, + retainedBytes: this.retainedBytes, + frames: this.frames.length, + transferring: this.transferring, + closed: this.closed, + bound: this.boundRange !== null + } + } +} diff --git a/src/main/runtime/rpc/terminal-source-range-registry.ts b/src/main/runtime/rpc/terminal-source-range-registry.ts new file mode 100644 index 00000000000..c2192183a87 --- /dev/null +++ b/src/main/runtime/rpc/terminal-source-range-registry.ts @@ -0,0 +1,43 @@ +import { TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION } from '../../../shared/terminal-multiplex-flow-control' +import { + TerminalSourceRangeLedger, + type TerminalSourceRangeBudget +} from './terminal-source-range-ledger' + +export const TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES = 16 * 1024 * 1024 + +export class TerminalSourceRangeRegistry { + private readonly ledgers = new Set() + private retainedBytes = 0 + + open(streamGeneration: string): TerminalSourceRangeLedger | null { + if (this.ledgers.size >= TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION) { + return null + } + let ledger: TerminalSourceRangeLedger + const budget: TerminalSourceRangeBudget = { + canReserve: (bytes) => + this.retainedBytes + bytes <= TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES, + reserve: (bytes) => { + if (this.retainedBytes + bytes > TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES) { + return false + } + this.retainedBytes += bytes + return true + }, + release: (bytes) => { + this.retainedBytes = Math.max(0, this.retainedBytes - bytes) + }, + close: () => { + this.ledgers.delete(ledger) + } + } + ledger = new TerminalSourceRangeLedger(streamGeneration, budget) + this.ledgers.add(ledger) + return ledger + } + + getDebugSnapshot() { + return { streams: this.ledgers.size, retainedBytes: this.retainedBytes } + } +} diff --git a/src/main/runtime/rpc/terminal-source-range-validation.ts b/src/main/runtime/rpc/terminal-source-range-validation.ts new file mode 100644 index 00000000000..1aed736a310 --- /dev/null +++ b/src/main/runtime/rpc/terminal-source-range-validation.ts @@ -0,0 +1,99 @@ +import { + assertTerminalOutputSourceRange, + sameTerminalOutputSourceIdentity, + type TerminalOutputSourceRange +} from '../../../shared/terminal-output-source-range' + +export type TerminalSourceRangeFrame = Readonly<{ + encodedStartByte: number + encodedEndByte: number + displayLength: number + outputSeq?: number + sourceRanges: readonly TerminalOutputSourceRange[] +}> + +export function freezeTerminalOutputSourceRanges( + ranges: readonly TerminalOutputSourceRange[] +): readonly TerminalOutputSourceRange[] { + return Object.freeze( + ranges.map((range) => + Object.freeze({ + ...range, + transform: Object.freeze({ ...range.transform }) + }) + ) + ) +} + +export function validateTerminalSourceRangeFrame( + displayLength: number, + ranges: readonly TerminalOutputSourceRange[] +): boolean { + if (!Number.isSafeInteger(displayLength) || displayLength < 0) { + return false + } + if (ranges.length === 0) { + return true + } + try { + for (const range of ranges) { + assertTerminalOutputSourceRange(range) + } + } catch { + return false + } + const first = ranges[0]! + let previous = first + for (const range of ranges.slice(1)) { + if ( + !sameTerminalOutputSourceIdentity(first, range) || + range.sourceStartSu !== previous.sourceEndSu || + range.displayStart !== previous.displayEnd + ) { + return false + } + previous = range + } + return previous.displayEnd - first.displayStart === displayLength +} + +export function replaceTerminalSourceRangeFrames( + frames: readonly TerminalSourceRangeFrame[], + snapshotSeq: number +): Readonly<{ + frames: TerminalSourceRangeFrame[] + mappingMode: 'mapped' | null + boundRange: TerminalOutputSourceRange | null + mappedSourceEndSu: number | null + mappedDisplayEnd: number | null +}> { + const replaced = frames.map((frame) => + typeof frame.outputSeq === 'number' && frame.outputSeq <= snapshotSeq + ? Object.freeze({ ...frame, sourceRanges: Object.freeze([]) }) + : frame + ) + const remainingRanges = replaced.flatMap((frame) => frame.sourceRanges) + const last = remainingRanges.at(-1) + return Object.freeze({ + frames: replaced, + mappingMode: remainingRanges.length > 0 ? 'mapped' : null, + boundRange: last ?? null, + mappedSourceEndSu: last?.sourceEndSu ?? null, + mappedDisplayEnd: last?.displayEnd ?? null + }) +} + +export function canPlanTerminalSourceRangeReplacement( + frames: readonly TerminalSourceRangeFrame[], + snapshotSeq: number +): boolean { + return ( + Number.isSafeInteger(snapshotSeq) && + snapshotSeq >= 0 && + frames.every( + (frame) => + frame.sourceRanges.length === 0 || + (typeof frame.outputSeq === 'number' && frame.outputSeq <= snapshotSeq) + ) + ) +} diff --git a/src/main/ssh/relay-frame-decoder.ts b/src/main/ssh/relay-frame-decoder.ts new file mode 100644 index 00000000000..6ee4d2f2b14 --- /dev/null +++ b/src/main/ssh/relay-frame-decoder.ts @@ -0,0 +1,264 @@ +import { + containFrameDecoderContinuation, + publishFrameDecoderError, + type DecodedFrame, + type FrameDecoderOptions +} from '../../shared/relay-frame-decoder-contract' +import { RelayFrameBuffer } from '../../shared/relay-frame-buffer' +export { + FrameDecoderContinuationError, + type DecodedFrame, + type FrameDecoderOptions +} from '../../shared/relay-frame-decoder-contract' + +export const HEADER_LENGTH = 13 +export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 +export const FRAME_DECODER_MAX_FRAMES_PER_TURN = 64 +export const FRAME_DECODER_MAX_BYTES_PER_TURN = MAX_MESSAGE_SIZE + HEADER_LENGTH +export const FRAME_DECODER_MAX_TURN_MS = 4, + FRAME_DECODER_MAX_RETAINED_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024 + +export class FrameDecoder { + private readonly buffer = new RelayFrameBuffer() + private oversizedPayloadBytesRemaining = 0 + private onFrame: (frame: DecodedFrame) => void + private onError: ((err: Error) => void) | null + private maxFramesPerTurn: number + private maxBytesPerTurn: number + private maxTurnMs: number + private now: () => number + private schedule: (callback: () => void) => unknown + private cancelScheduled: (handle: unknown) => void + private pause: (() => void) | null + private resume: (() => void) | null + private continuationHandle: unknown + private continuationHandleAssigned = false + private continuationScheduled = false + private paused = false + private draining = false + private generation = 0 + + constructor( + onFrame: (frame: DecodedFrame) => void, + onError?: (err: Error) => void, + options: FrameDecoderOptions = {} + ) { + this.onFrame = onFrame + this.onError = onError ?? null + this.maxFramesPerTurn = positiveLimit( + options.maxFramesPerTurn, + FRAME_DECODER_MAX_FRAMES_PER_TURN + ) + this.maxBytesPerTurn = positiveLimit(options.maxBytesPerTurn, FRAME_DECODER_MAX_BYTES_PER_TURN) + this.maxTurnMs = positiveLimit(options.maxTurnMs, FRAME_DECODER_MAX_TURN_MS) + this.now = options.now ?? Date.now + this.schedule = options.schedule ?? ((callback) => setImmediate(callback)) + this.cancelScheduled = + options.cancelScheduled ?? ((handle) => clearImmediate(handle as NodeJS.Immediate)) + this.pause = options.pause ?? null + this.resume = options.resume ?? null + } + + feed(chunk: Buffer | Uint8Array): void { + const buf = Buffer.isBuffer(chunk) + ? chunk + : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) + const retained = this.buffer.length + buf.length + if (retained > FRAME_DECODER_MAX_RETAINED_BYTES) { + this.reset() + publishFrameDecoderError( + this.onError, + new Error(`Frame decoder retained-input limit exceeded: ${retained}`) + ) + return + } + if (buf.length > 0) { + this.buffer.append(buf) + } + if (!this.draining && !this.continuationScheduled) { + this.drainTurn() + } + } + + reset(): void { + this.generation += 1 + this.cancelContinuation() + this.buffer.clear() + this.oversizedPayloadBytesRemaining = 0 + this.releasePause() + } + + drain(): Buffer { + const out = this.buffer.drain() + this.reset() + return out + } + + private drainTurn(): void { + if (this.draining) { + return + } + this.draining = true + const generation = this.generation + const startedAt = this.now() + let frames = 0 + let bytes = 0 + + try { + while (generation === this.generation) { + if ( + frames >= this.maxFramesPerTurn || + bytes >= this.maxBytesPerTurn || + (frames > 0 && this.now() - startedAt >= this.maxTurnMs) + ) { + break + } + const discarded = this.discardOversizedPayload(bytes) + if (discarded > 0) { + bytes += discarded + continue + } + if (this.buffer.length < HEADER_LENGTH) { + break + } + const header = this.buffer.peek(HEADER_LENGTH) + const length = header.readUInt32BE(9) + if (length > MAX_MESSAGE_SIZE) { + this.buffer.discard(HEADER_LENGTH) + this.oversizedPayloadBytesRemaining = length + bytes += HEADER_LENGTH + publishFrameDecoderError( + this.onError, + new Error(`Frame payload too large: ${length} bytes — discarded`) + ) + continue + } + const totalLength = HEADER_LENGTH + length + if (this.buffer.length < totalLength) { + break + } + if (frames > 0 && bytes + totalLength > this.maxBytesPerTurn) { + break + } + const framed = this.buffer.take(totalLength) + frames += 1 + bytes += totalLength + this.onFrame({ + type: framed[0], + id: framed.readUInt32BE(1), + ack: framed.readUInt32BE(5), + payload: framed.subarray(HEADER_LENGTH, totalLength) + }) + } + } finally { + this.draining = false + } + + if (generation !== this.generation) { + return + } + if (this.hasRunnableWork()) { + this.scheduleContinuation() + } else { + this.releasePause() + } + } + + private discardOversizedPayload(bytes: number): number { + if (this.oversizedPayloadBytesRemaining === 0 || this.buffer.length === 0) { + return 0 + } + const discarded = Math.min( + this.oversizedPayloadBytesRemaining, + this.buffer.length, + Math.max(1, this.maxBytesPerTurn - bytes) + ) + this.buffer.discard(discarded) + this.oversizedPayloadBytesRemaining -= discarded + return discarded + } + + private hasRunnableWork(): boolean { + if (this.oversizedPayloadBytesRemaining > 0) { + return this.buffer.length > 0 + } + if (this.buffer.length < HEADER_LENGTH) { + return false + } + const length = this.buffer.peek(HEADER_LENGTH).readUInt32BE(9) + return length > MAX_MESSAGE_SIZE || this.buffer.length >= HEADER_LENGTH + length + } + + private scheduleContinuation(): void { + if (this.continuationScheduled) { + return + } + const generation = this.generation + this.continuationScheduled = true + try { + this.acquirePause() + } catch (error) { + this.continuationScheduled = false + throw error + } + if (generation !== this.generation) { + this.continuationScheduled = false + return + } + try { + this.continuationHandle = this.schedule(() => { + if (!this.continuationScheduled || generation !== this.generation) { + return + } + this.continuationScheduled = false + this.continuationHandleAssigned = false + this.continuationHandle = undefined + try { + this.drainTurn() + } catch (error) { + containFrameDecoderContinuation(() => this.reset(), this.onError, error) + } + }) + this.continuationHandleAssigned = true + } catch (error) { + this.continuationScheduled = false + this.continuationHandle = undefined + this.releasePause() + throw error + } + } + + private cancelContinuation(): void { + if (!this.continuationScheduled) { + return + } + this.continuationScheduled = false + if (this.continuationHandleAssigned) { + this.cancelScheduled(this.continuationHandle) + } + this.continuationHandleAssigned = false + this.continuationHandle = undefined + } + + private acquirePause(): void { + if (!this.paused) { + this.paused = true + try { + this.pause?.() + } catch (error) { + this.paused = false + throw error + } + } + } + + private releasePause(): void { + if (this.paused) { + this.paused = false + this.resume?.() + } + } +} + +const positiveLimit = (value: number | undefined, fallback: number): number => + value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback diff --git a/src/main/ssh/relay-protocol-backpressure.test.ts b/src/main/ssh/relay-protocol-backpressure.test.ts new file mode 100644 index 00000000000..a1ee62655e8 --- /dev/null +++ b/src/main/ssh/relay-protocol-backpressure.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it, vi } from 'vitest' +import { + FrameDecoder, + FrameDecoderContinuationError, + FRAME_DECODER_MAX_RETAINED_BYTES, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + MessageType, + encodeFrame, + type DecodedFrame +} from './relay-protocol' + +function createScheduler(): { + schedule: (callback: () => void) => number + cancel: (handle: unknown) => void + runNext: () => void + pending: () => number +} { + let nextHandle = 1 + const callbacks = new Map void>() + return { + schedule: (callback) => { + const handle = nextHandle++ + callbacks.set(handle, callback) + return handle + }, + cancel: (handle) => callbacks.delete(handle as number), + runNext: () => { + const entry = callbacks.entries().next().value as [number, () => void] | undefined + if (!entry) { + throw new Error('No decoder continuation scheduled') + } + callbacks.delete(entry[0]) + entry[1]() + }, + pending: () => callbacks.size + } +} + +function frame(id: number, payload = `${id}`): Buffer { + return encodeFrame(MessageType.Regular, id, 0, Buffer.from(payload)) +} + +describe('FrameDecoder bounded turns', () => { + it('retains at most one maximum frame plus one MiB of partial input', () => { + expect(FRAME_DECODER_MAX_RETAINED_BYTES).toBe(MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024) + const maximumFrame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(MAX_MESSAGE_SIZE)) + const acceptedError = vi.fn() + const accepted = new FrameDecoder(vi.fn(), acceptedError) + + accepted.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024)])) + + expect(acceptedError).not.toHaveBeenCalled() + expect(accepted.drain()).toHaveLength(1024 * 1024) + + const excessError = vi.fn() + const excess = new FrameDecoder(vi.fn(), excessError) + excess.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024 + 1)])) + + expect(excessError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('retained-input') }) + ) + expect(excess.drain()).toHaveLength(0) + }) + + it('fails closed when one delivered chunk exceeds retained-input capacity', () => { + const onError = vi.fn() + const decoder = new FrameDecoder(vi.fn(), onError) + + decoder.feed(Buffer.alloc(FRAME_DECODER_MAX_RETAINED_BYTES + 1)) + + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('retained-input') }) + ) + expect(decoder.drain()).toHaveLength(0) + }) + + it('emits the first frame synchronously and preserves order through self-pause', () => { + const scheduler = createScheduler() + const seen: number[] = [] + let decoder: FrameDecoder + const pause = vi.fn(() => decoder.feed(frame(4))) + const resume = vi.fn() + decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, { + maxFramesPerTurn: 1, + schedule: scheduler.schedule, + cancelScheduled: scheduler.cancel, + pause, + resume + }) + + decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)])) + + expect(seen).toEqual([1]) + expect(pause).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(1) + + scheduler.runNext() + scheduler.runNext() + scheduler.runNext() + + expect(seen).toEqual([1, 2, 3, 4]) + expect(pause).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(0) + }) + + it('bounds decoded bytes and time independently from the frame count', () => { + const byteScheduler = createScheduler() + const byteSeen: number[] = [] + const first = frame(1, 'one') + const second = frame(2, 'two') + const byteDecoder = new FrameDecoder((decoded) => byteSeen.push(decoded.id), undefined, { + maxFramesPerTurn: 64, + maxBytesPerTurn: first.length, + schedule: byteScheduler.schedule, + cancelScheduled: byteScheduler.cancel + }) + + byteDecoder.feed(Buffer.concat([first, second])) + expect(byteSeen).toEqual([1]) + byteScheduler.runNext() + expect(byteSeen).toEqual([1, 2]) + + const timeScheduler = createScheduler() + const timeSeen: number[] = [] + let nowCalls = 0 + const timeDecoder = new FrameDecoder((decoded) => timeSeen.push(decoded.id), undefined, { + maxFramesPerTurn: 64, + maxBytesPerTurn: MAX_MESSAGE_SIZE + HEADER_LENGTH, + maxTurnMs: 4, + now: () => (nowCalls++ === 0 ? 0 : 5), + schedule: timeScheduler.schedule, + cancelScheduled: timeScheduler.cancel + }) + + timeDecoder.feed(Buffer.concat([frame(3), frame(4)])) + expect(timeSeen).toEqual([3]) + timeScheduler.runNext() + expect(timeSeen).toEqual([3, 4]) + }) + + it('releases its pause epoch when continuation scheduling throws', () => { + const pause = vi.fn() + const resume = vi.fn() + const decoder = new FrameDecoder(() => {}, undefined, { + maxFramesPerTurn: 1, + pause, + resume, + schedule: () => { + throw new Error('scheduler unavailable') + } + }) + + expect(() => decoder.feed(Buffer.concat([frame(1), frame(2)]))).toThrow('scheduler unavailable') + expect(pause).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + decoder.reset() + expect(resume).toHaveBeenCalledTimes(1) + }) + + it('contains a throwing continuation, resets residue, and reports one typed error', () => { + const scheduler = createScheduler() + const seen: number[] = [] + const onError = vi.fn() + const pause = vi.fn() + const resume = vi.fn() + const decoder = new FrameDecoder( + (decoded) => { + if (decoded.id === 2) { + throw new Error('frame owner failed') + } + seen.push(decoded.id) + }, + onError, + { + maxFramesPerTurn: 1, + schedule: scheduler.schedule, + cancelScheduled: scheduler.cancel, + pause, + resume + } + ) + + decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)])) + expect(() => scheduler.runNext()).not.toThrow() + + expect(seen).toEqual([1]) + expect(onError).toHaveBeenCalledExactlyOnceWith(expect.any(FrameDecoderContinuationError)) + expect(onError.mock.calls[0]?.[0]).toMatchObject({ + name: 'FrameDecoderContinuationError', + cause: expect.objectContaining({ message: 'frame owner failed' }) + }) + expect(pause).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(0) + expect(decoder.drain()).toHaveLength(0) + + decoder.feed(frame(4)) + expect(seen).toEqual([1, 4]) + }) + + it('keeps reads active for partial frames and incrementally discards oversized payloads', () => { + const errors: Error[] = [] + const seen: DecodedFrame[] = [] + const pause = vi.fn() + const decoder = new FrameDecoder( + (decoded) => seen.push(decoded), + (error) => errors.push(error), + { pause } + ) + const valid = frame(2, 'complete') + + decoder.feed(valid.subarray(0, HEADER_LENGTH + 2)) + expect(seen).toHaveLength(0) + expect(pause).not.toHaveBeenCalled() + decoder.feed(valid.subarray(HEADER_LENGTH + 2)) + expect(seen.map(({ id }) => id)).toEqual([2]) + + const oversizedHeader = Buffer.alloc(HEADER_LENGTH) + oversizedHeader[0] = MessageType.Regular + oversizedHeader.writeUInt32BE(3, 1) + oversizedHeader.writeUInt32BE(MAX_MESSAGE_SIZE + 1, 9) + decoder.feed(Buffer.concat([oversizedHeader, Buffer.alloc(32)])) + + expect(errors).toHaveLength(1) + expect(pause).not.toHaveBeenCalled() + decoder.reset() + decoder.feed(frame(4, 'after-reset')) + expect(seen.map(({ id }) => id)).toEqual([2, 4]) + }) + + it('drain and reset cancel continuation ownership without replaying residue', () => { + const scheduler = createScheduler() + const seen: number[] = [] + const resume = vi.fn() + const cancel = vi.fn(scheduler.cancel) + const decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, { + maxFramesPerTurn: 1, + schedule: scheduler.schedule, + cancelScheduled: cancel, + resume + }) + const second = frame(2, 'residue') + + decoder.feed(Buffer.concat([frame(1), second])) + const residue = decoder.drain() + + expect(seen).toEqual([1]) + expect(residue.equals(second)).toBe(true) + expect(cancel).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(0) + + decoder.feed(Buffer.concat([frame(3), frame(4)])) + decoder.reset() + expect(cancel).toHaveBeenCalledTimes(2) + expect(scheduler.pending()).toBe(0) + + decoder.feed(frame(5)) + expect(seen).toEqual([1, 3, 5]) + }) +}) diff --git a/src/main/ssh/relay-protocol.ts b/src/main/ssh/relay-protocol.ts index 0412c97994d..4a60237874a 100644 --- a/src/main/ssh/relay-protocol.ts +++ b/src/main/ssh/relay-protocol.ts @@ -3,17 +3,34 @@ // See design-ssh-support.md § JSON-RPC Protocol Specification. import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../../shared/ssh-types' +import { + FrameDecoder, + FrameDecoderContinuationError, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + FRAME_DECODER_MAX_FRAMES_PER_TURN, + FRAME_DECODER_MAX_BYTES_PER_TURN, + FRAME_DECODER_MAX_TURN_MS, + FRAME_DECODER_MAX_RETAINED_BYTES +} from './relay-frame-decoder' + +export { + FrameDecoder, + FrameDecoderContinuationError, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + FRAME_DECODER_MAX_FRAMES_PER_TURN, + FRAME_DECODER_MAX_BYTES_PER_TURN, + FRAME_DECODER_MAX_TURN_MS, + FRAME_DECODER_MAX_RETAINED_BYTES +} +export type { DecodedFrame, FrameDecoderOptions } from './relay-frame-decoder' export const RELAY_VERSION = '0.1.0' export const RELAY_SENTINEL = `ORCA-RELAY v${RELAY_VERSION} READY\n` export const RELAY_SENTINEL_TIMEOUT_MS = 10_000 export const RELAY_REMOTE_DIR = '.orca-remote' -// ── Framing constants (VS Code ProtocolConstants) ─────────────────── - -export const HEADER_LENGTH = 13 -export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 // 16 MB - /** Message type byte. */ export const MessageType = { Regular: 1, @@ -155,148 +172,6 @@ export function encodeKeepAliveFrame(id: number, ack: number): Buffer { return encodeFrame(MessageType.KeepAlive, id, ack, Buffer.alloc(0)) } -export type DecodedFrame = { - type: number - id: number - ack: number - payload: Buffer -} - -/** - * Incremental frame parser. Feed it chunks of data; it emits complete frames. - */ -export class FrameDecoder { - // Why: feed() runs on the Electron main thread for every SSH channel data - // event. Rebuilding one contiguous buffer per feed (Buffer.concat) re-copies - // every already-buffered byte for each incoming ~32KB TCP chunk — O(n²) per - // large frame (a 340KB fs.streamChunk frame cost ~2MB of memcpy). A chunk - // list assembles each frame exactly once instead. - private chunks: Buffer[] = [] - private bufferedLength = 0 - private onFrame: (frame: DecodedFrame) => void - private onError: ((err: Error) => void) | null - - constructor(onFrame: (frame: DecodedFrame) => void, onError?: (err: Error) => void) { - this.onFrame = onFrame - this.onError = onError ?? null - } - - feed(chunk: Buffer | Uint8Array): void { - const buf = Buffer.isBuffer(chunk) - ? chunk - : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) - if (buf.length > 0) { - this.chunks.push(buf) - this.bufferedLength += buf.length - } - - while (this.bufferedLength >= HEADER_LENGTH) { - const header = this.peekBytes(HEADER_LENGTH) - const length = header.readUInt32BE(9) - const totalLength = HEADER_LENGTH + length - - if (this.bufferedLength < totalLength) { - // Not fully received yet (also holds oversized frames until they can - // be skipped whole, keeping the decoder synchronized). - break - } - - // Why: throwing here would leave the buffer in a partially consumed - // state — subsequent feed() calls would try to parse leftover payload - // bytes as a new header, corrupting every future frame. Instead we - // skip the entire oversized frame so the decoder stays synchronized. - if (length > MAX_MESSAGE_SIZE) { - this.discardBytes(totalLength) - const err = new Error(`Frame payload too large: ${length} bytes — discarded`) - if (this.onError) { - this.onError(err) - } - continue - } - - const framed = this.takeBytes(totalLength) - const frame: DecodedFrame = { - type: framed[0], - id: framed.readUInt32BE(1), - ack: framed.readUInt32BE(5), - payload: framed.subarray(HEADER_LENGTH, totalLength) - } - this.onFrame(frame) - } - } - - reset(): void { - this.chunks = [] - this.bufferedLength = 0 - } - - /** View of the first `count` buffered bytes without consuming them. */ - private peekBytes(count: number): Buffer { - const first = this.chunks[0] - if (first.length >= count) { - return first - } - const out = Buffer.allocUnsafe(count) - let copied = 0 - for (const part of this.chunks) { - copied += part.copy(out, copied, 0, Math.min(part.length, count - copied)) - if (copied >= count) { - break - } - } - return out - } - - /** Consume and return the first `count` buffered bytes (single copy). */ - private takeBytes(count: number): Buffer { - const first = this.chunks[0] - if (first.length === count) { - this.chunks.shift() - this.bufferedLength -= count - return first - } - if (first.length > count) { - this.chunks[0] = first.subarray(count) - this.bufferedLength -= count - return first.subarray(0, count) - } - const out = Buffer.allocUnsafe(count) - let copied = 0 - while (copied < count) { - const part = this.chunks[0] - const take = Math.min(part.length, count - copied) - part.copy(out, copied, 0, take) - copied += take - if (take === part.length) { - this.chunks.shift() - } else { - this.chunks[0] = part.subarray(take) - } - } - this.bufferedLength -= count - return out - } - - /** Consume the first `count` buffered bytes without assembling them. */ - private discardBytes(count: number): void { - let remaining = count - while (remaining > 0) { - const part = this.chunks[0] - if (part.length <= remaining) { - this.chunks.shift() - remaining -= part.length - } else { - this.chunks[0] = part.subarray(remaining) - remaining = 0 - } - } - this.bufferedLength -= count - } -} - -/** - * Parse a JSON-RPC message from a frame payload. - */ export function parseJsonRpcMessage(payload: Buffer): JsonRpcMessage { const text = payload.toString('utf-8') const msg = JSON.parse(text) as JsonRpcMessage diff --git a/src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts b/src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts new file mode 100644 index 00000000000..7f6100f6d75 --- /dev/null +++ b/src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts @@ -0,0 +1,203 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer' +import { encodeFrame, encodeKeepAliveFrame, HEADER_LENGTH, MessageType } from './relay-protocol' + +type MockTransport = MultiplexerTransport & { + data: (chunk: Buffer) => void + written: Buffer[] + pauseReads: ReturnType + resumeReads: ReturnType +} + +type MuxInternals = { + nextOutgoingSeq: number + highestAckedBySelf: number + lastReceivedAt: number + decoderReadPaused: boolean + unackedTimestamps: Map +} + +function createTransport(): MockTransport { + let onData: (chunk: Buffer) => void = () => {} + const written: Buffer[] = [] + const pauseReads = vi.fn() + const resumeReads = vi.fn() + return { + write: (data) => { + written.push(data) + }, + onData: (callback) => { + onData = callback + }, + onClose: () => {}, + pauseReads, + resumeReads, + data: (chunk) => onData(chunk), + written + } +} + +function internals(mux: SshChannelMultiplexer): MuxInternals { + return mux as unknown as MuxInternals +} + +describe('SshChannelMultiplexer backpressure hardening', () => { + let transport: MockTransport + let mux: SshChannelMultiplexer + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + transport = createTransport() + mux = new SshChannelMultiplexer(transport) + }) + + afterEach(() => { + mux.dispose() + vi.restoreAllMocks() + vi.useRealTimers() + }) + + it('clamps ack=0xffffffff and deletes only retained sequence keys', () => { + mux.notify('one') + mux.notify('two') + const state = internals(mux) + const deleteSpy = vi.spyOn(state.unackedTimestamps, 'delete') + + transport.data(encodeKeepAliveFrame(1, 0xffffffff)) + + expect(deleteSpy).toHaveBeenCalledTimes(2) + expect(state.unackedTimestamps.size).toBe(0) + expect(state.highestAckedBySelf).toBe(state.nextOutgoingSeq - 1) + }) + + it('caps timestamps while preserving a reserved liveness entry', () => { + for (let i = 0; i < 5000; i += 1) { + mux.notify('bounded') + } + const state = internals(mux) + + expect(state.unackedTimestamps.size).toBe(4095) + expect(state.unackedTimestamps.has(1)).toBe(true) + void mux.probeLiveness(60_000) + expect(state.unackedTimestamps.size).toBe(4096) + void mux.probeLiveness(60_000) + expect(state.unackedTimestamps.size).toBe(4096) + }) + + it('suppresses timeout while self-paused and rebases both clocks on resume', () => { + mux.dispose() + const continuations: (() => void)[] = [] + vi.spyOn(globalThis, 'setImmediate').mockImplementation( + (callback: (...args: never[]) => void) => { + continuations.push(callback) + return {} as NodeJS.Immediate + } + ) + transport = createTransport() + mux = new SshChannelMultiplexer(transport) + mux.notify('tracked') + + const incoming = Buffer.concat( + Array.from({ length: 65 }, (_, index) => encodeKeepAliveFrame(index + 1, 0)) + ) + transport.data(incoming) + const state = internals(mux) + + expect(state.decoderReadPaused).toBe(true) + expect(transport.pauseReads).toHaveBeenCalledTimes(1) + expect(continuations).toHaveLength(1) + + vi.advanceTimersByTime(25_000) + expect(mux.isDisposed()).toBe(false) + + continuations.shift()!() + const resumedAt = Date.now() + expect(state.decoderReadPaused).toBe(false) + expect(transport.resumeReads).toHaveBeenCalledTimes(1) + expect(state.lastReceivedAt).toBe(resumedAt) + expect(new Set(state.unackedTimestamps.values())).toEqual(new Set([resumedAt])) + }) + + it('keeps frame dispatch ordered across a decoder continuation', () => { + const seen: number[] = [] + mux.onNotification((_method, params) => seen.push(params.index as number)) + const frames = Array.from({ length: 65 }, (_, index) => { + const payload = Buffer.from( + JSON.stringify({ jsonrpc: '2.0', method: 'ordered', params: { index } }) + ) + return encodeFrame(MessageType.Regular, index + 1, 0, payload) + }) + + transport.data(Buffer.concat(frames)) + expect(seen).toEqual(Array.from({ length: 64 }, (_, index) => index)) + vi.runAllTicks() + vi.advanceTimersByTime(0) + expect(seen).toEqual(Array.from({ length: 65 }, (_, index) => index)) + }) + + it('prioritizes source ACK and control while preserving FIFO and ordinary progress', async () => { + mux.dispose() + const written: Buffer[] = [] + let drain = (): void => {} + let deliver = (_data: Buffer): void => {} + transport = { + write: (data) => { + written.push(data) + return written.length !== 1 + }, + onDrain: (callback) => { + drain = callback + }, + onData: (callback) => { + deliver = callback + }, + onClose: vi.fn(), + pauseReads: vi.fn<() => void>(), + resumeReads: vi.fn<() => void>(), + data: vi.fn<(chunk: Buffer) => void>(), + written, + close: vi.fn() + } + mux = new SshChannelMultiplexer(transport) + mux.onRequest('client.control', () => ({ accepted: true })) + const controller = new AbortController() + + mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-1' }) + mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-2' }) + mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-3' }) + mux.notify('pty.ackData', { acknowledgements: [] }) + const request = mux.request('fs.scan', {}, { signal: controller.signal }) + controller.abort() + mux.notify('pty.exit', { id: 'pty-1', code: 0 }) + const remoteRequest = Buffer.from( + JSON.stringify({ jsonrpc: '2.0', id: 91, method: 'client.control' }) + ) + deliver(encodeFrame(MessageType.Regular, 1, 0, remoteRequest)) + await Promise.resolve() + await Promise.resolve() + + expect(written).toHaveLength(1) + drain() + const payloads = written.map((frame) => + JSON.parse(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + frame.readUInt32BE(9)).toString()) + ) + expect( + payloads.map((payload) => + payload.method === 'pty.data' + ? `pty.data:${payload.params.data}` + : (payload.method ?? `response:${payload.id}`) + ) + ).toEqual([ + 'pty.data:ordinary-1', + 'pty.ackData', + 'fs.scan', + 'rpc.cancel', + 'pty.exit', + 'pty.data:ordinary-2', + 'response:91', + 'pty.data:ordinary-3' + ]) + await expect(request).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/src/main/ssh/ssh-channel-multiplexer-settlement.test.ts b/src/main/ssh/ssh-channel-multiplexer-settlement.test.ts new file mode 100644 index 00000000000..272f84399b3 --- /dev/null +++ b/src/main/ssh/ssh-channel-multiplexer-settlement.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from 'vitest' +import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer' + +function transportHarness(): { + transport: MultiplexerTransport + settlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[] +} { + const settlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[] = [] + return { + transport: { + write: (_data, onSettled) => { + if (onSettled) { + settlements.push(onSettled) + } + }, + supportsWriteSettlement: true, + onData: vi.fn(), + onClose: vi.fn() + }, + settlements + } +} + +describe('SshChannelMultiplexer notification settlement', () => { + it('reports publication only from the transport write callback', () => { + const harness = transportHarness() + const mux = new SshChannelMultiplexer(harness.transport) + const settled = vi.fn() + + mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled) + expect(settled).not.toHaveBeenCalled() + harness.settlements[0]({ ok: true }) + expect(settled).toHaveBeenCalledWith({ ok: true }) + mux.dispose() + }) + + it('reports a synchronous write failure without publishing success', () => { + const error = new Error('write failed') + const mux = new SshChannelMultiplexer({ + write: () => { + throw error + }, + supportsWriteSettlement: true, + onData: vi.fn(), + onClose: vi.fn() + }) + const settled = vi.fn() + + mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled) + expect(settled).toHaveBeenCalledWith({ ok: false, error }) + expect(mux.isDisposed()).toBe(true) + }) + + it('settles once when a hostile transport invokes its callback and then throws', () => { + const mux = new SshChannelMultiplexer({ + write: (_data, onSettled) => { + onSettled?.({ ok: true }) + throw new Error('late throw') + }, + supportsWriteSettlement: true, + onData: vi.fn(), + onClose: vi.fn() + }) + const settled = vi.fn() + + mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled) + expect(settled).toHaveBeenCalledOnce() + expect(settled).toHaveBeenCalledWith({ ok: true }) + }) + + it('fails an unsettled publication when the multiplexer is disposed', () => { + const close = vi.fn() + const mux = new SshChannelMultiplexer({ + write: () => false, + supportsWriteSettlement: true, + onDrain: vi.fn(), + onData: vi.fn(), + onClose: vi.fn(), + close + }) + const settled = vi.fn() + + mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled) + expect(settled).not.toHaveBeenCalled() + mux.dispose() + + expect(settled).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ code: 'DISPOSED' }) + }) + expect(close).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ssh/ssh-channel-multiplexer.test.ts b/src/main/ssh/ssh-channel-multiplexer.test.ts index d17e756f75b..f58e4172787 100644 --- a/src/main/ssh/ssh-channel-multiplexer.test.ts +++ b/src/main/ssh/ssh-channel-multiplexer.test.ts @@ -12,7 +12,9 @@ function createMockTransport(): MultiplexerTransport & { const written: Buffer[] = [] return { - write: (data: Buffer) => written.push(data), + write: (data: Buffer) => { + written.push(data) + }, onData: (cb) => dataCallbacks.push(cb), onClose: (cb) => closeCallbacks.push(cb), dataCallbacks, @@ -69,6 +71,7 @@ type MuxInternals = { disposeHandlers: unknown[] lastReceivedAt: number unackedTimestamps: Map + writerSaturated: boolean } function getMuxInternals(instance: SshChannelMultiplexer): MuxInternals { @@ -123,6 +126,28 @@ describe('SshChannelMultiplexer', () => { await expect(promise).rejects.toThrow('PTY allocation failed') }) + it('runs beforeResolve before an adjacent notification in the same decoder turn', async () => { + const order: string[] = [] + mux.onNotification(() => order.push('notification')) + const promise = mux.request( + 'pty.attach', + { id: 'pty-1' }, + { + beforeResolve: () => order.push('beforeResolve') + } + ) + + transport.dataCallbacks[0]( + Buffer.concat([ + makeResponseFrame(1, { incarnationId: 'incarnation-1' }, 1), + makeNotificationFrame('pty.data', { id: 'pty-1', data: 'first' }, 2) + ]) + ) + + expect(order).toEqual(['beforeResolve', 'notification']) + await expect(promise).resolves.toEqual({ incarnationId: 'incarnation-1' }) + }) + it('times out after 30s with no response', async () => { const promise = mux.request('pty.spawn') @@ -328,6 +353,43 @@ describe('SshChannelMultiplexer', () => { vi.advanceTimersByTime(25_000) expect(mux.isDisposed()).toBe(true) }) + + it('suppresses false death while locally saturated and rebases both clocks on drain', () => { + mux.dispose() + let drain = (): void => {} + const written: Buffer[] = [] + const saturatedTransport: MultiplexerTransport = { + write: (data) => { + written.push(data) + return false + }, + supportsWriteSettlement: true, + onDrain: (callback) => { + drain = callback + }, + onData: vi.fn(), + onClose: vi.fn() + } + mux = new SshChannelMultiplexer(saturatedTransport) + + vi.advanceTimersByTime(5_000) + expect(getMuxInternals(mux).writerSaturated).toBe(true) + vi.advanceTimersByTime(25_000) + expect(mux.isDisposed()).toBe(false) + expect(written).toHaveLength(1) + + drain() + const resumedAt = Date.now() + const internals = getMuxInternals(mux) + expect(internals.writerSaturated).toBe(false) + expect(internals.lastReceivedAt).toBe(resumedAt) + expect(new Set(internals.unackedTimestamps.values())).toEqual(new Set([resumedAt])) + + vi.advanceTimersByTime(20_000) + expect(mux.isDisposed()).toBe(false) + vi.advanceTimersByTime(5_000) + expect(mux.isDisposed()).toBe(true) + }) }) describe('wake guard (timer pause across system sleep, #7773)', () => { diff --git a/src/main/ssh/ssh-channel-multiplexer.ts b/src/main/ssh/ssh-channel-multiplexer.ts index 9886b724755..575decba730 100644 --- a/src/main/ssh/ssh-channel-multiplexer.ts +++ b/src/main/ssh/ssh-channel-multiplexer.ts @@ -14,26 +14,36 @@ import { type JsonRpcResponse, type JsonRpcNotification } from './relay-protocol' +import { + SshMultiplexerTransportWriter, + type MultiplexerTransport, + type MultiplexerWriteSettlement, + type MultiplexerWriterLane +} from './ssh-multiplexer-transport-writer' -export type MultiplexerTransport = { - write: (data: Buffer) => void - onData: (cb: (data: Buffer) => void) => void - onClose: (cb: () => void) => void - close?: () => void -} +export type { MultiplexerTransport, MultiplexerWriteSettlement } type PendingRequest = { resolve: (result: unknown) => void reject: (error: Error) => void + beforeResolve?: (result: unknown) => void timer: ReturnType cleanup: () => void } +export type SshMultiplexerRequestOptions = { + signal?: AbortSignal + timeoutMs?: number + beforeResolve?: (result: unknown) => void +} + export type NotificationHandler = (method: string, params: Record) => void export type MethodNotificationHandler = (params: Record) => void export type RequestHandler = (params: Record) => Promise | unknown const REQUEST_TIMEOUT_MS = 30_000 +const MAX_ORDINARY_UNACKED_TIMESTAMPS = 4095 +const MAX_UNACKED_TIMESTAMPS = MAX_ORDINARY_UNACKED_TIMESTAMPS + 1 // Why: a tick gap far beyond the interval means the process was paused // (system sleep, App Nap timer throttling) — not that the link is dead (#7773). const WAKE_GAP_MS = KEEPALIVE_SEND_MS * 3 @@ -41,6 +51,7 @@ const WAKE_GAP_MS = KEEPALIVE_SEND_MS * 3 export class SshChannelMultiplexer { private decoder: FrameDecoder private transport: MultiplexerTransport + private writer: SshMultiplexerTransportWriter private nextRequestId = 1 private nextOutgoingSeq = 1 private highestReceivedSeq = 0 @@ -56,6 +67,8 @@ export class SshChannelMultiplexer { private disposeHandlers: ((reason: 'shutdown' | 'connection_lost') => void)[] = [] private connectionHealthTimer: ReturnType | null = null private disposed = false + private decoderReadPaused = false + private writerSaturated = false // Track the oldest unacked outgoing message timestamp private unackedTimestamps = new Map() @@ -66,10 +79,19 @@ export class SshChannelMultiplexer { constructor(transport: MultiplexerTransport) { this.transport = transport + this.writer = new SshMultiplexerTransportWriter( + transport, + (error) => this.handleProtocolError(error), + (saturated) => this.handleWriterSaturationChange(saturated) + ) this.decoder = new FrameDecoder( (frame) => this.handleFrame(frame), - (err) => this.handleProtocolError(err) + (err) => this.handleProtocolError(err), + { + pause: () => this.pauseDecoderReads(), + resume: () => this.resumeDecoderReads() + } ) transport.onData((data) => { @@ -158,7 +180,7 @@ export class SshChannelMultiplexer { async request( method: string, params?: Record, - options?: { signal?: AbortSignal; timeoutMs?: number } + options?: SshMultiplexerRequestOptions ): Promise { if (this.disposed) { throw new Error('Multiplexer disposed') @@ -215,7 +237,13 @@ export class SshChannelMultiplexer { if (options?.signal) { options.signal.addEventListener('abort', onAbort, { once: true }) } - this.pendingRequests.set(id, { resolve, reject, timer, cleanup }) + this.pendingRequests.set(id, { + resolve, + reject, + beforeResolve: options?.beforeResolve, + timer, + cleanup + }) this.sendMessage(msg) }) } @@ -237,6 +265,25 @@ export class SshChannelMultiplexer { this.sendMessage(msg) } + notifyWithSettlement( + method: string, + params: Record | undefined, + onSettled: (result: { ok: true } | { ok: false; error: Error }) => void + ): void { + if (this.disposed) { + onSettled({ ok: false, error: new Error('Multiplexer disposed') }) + return + } + this.sendMessage( + { + jsonrpc: '2.0', + method, + ...(params !== undefined ? { params } : {}) + }, + onSettled + ) + } + /** * Send a fresh keepalive and resolve true when any frame arrives before the * timeout. Used on system resume to distinguish a link that survived sleep @@ -297,6 +344,9 @@ export class SshChannelMultiplexer { this.pendingRequests.delete(id) } + const writerError = new Error(errorMessage) as Error & { code: string } + writerError.code = errorCode + this.writer.dispose(writerError) this.unackedTimestamps.clear() // Why: relay teardown can race with late provider registration; disposed // muxes must not retain provider/session closures through subscribers. @@ -321,34 +371,27 @@ export class SshChannelMultiplexer { // ── Private ─────────────────────────────────────────────────────── - private sendMessage(msg: JsonRpcMessage): void { + private sendMessage( + msg: JsonRpcMessage, + onSettled?: (result: MultiplexerWriteSettlement) => void + ): void { const seq = this.nextOutgoingSeq++ const frame = encodeJsonRpcFrame(msg, seq, this.highestReceivedSeq) - this.unackedTimestamps.set(seq, Date.now()) - try { - this.transport.write(frame) - } catch (err) { - // Why: a remote reboot can make the SSH channel's stdin throw EPIPE - // from a timer/request path. Scope it to this mux instead of letting - // the Electron main process treat it as an uncaught exception. - this.handleProtocolError(err) - } + this.trackOutgoingTimestamp(seq, false) + this.writer.enqueue(frame, messageLane(msg), onSettled) } private sendKeepAlive(): void { if (this.disposed) { return } - const seq = this.nextOutgoingSeq++ + const seq = this.nextOutgoingSeq const frame = encodeKeepAliveFrame(seq, this.highestReceivedSeq) - this.unackedTimestamps.set(seq, Date.now()) - try { - this.transport.write(frame) - } catch (err) { - // Why: keepalive runs on an interval; without catching transport - // write failures here, a dead SSH host can terminate the whole app. - this.handleProtocolError(err) + if (!this.writer.enqueue(frame, 'liveness')) { + return } + this.nextOutgoingSeq++ + this.trackOutgoingTimestamp(seq, true) } private handleFrame(frame: DecodedFrame): void { @@ -363,12 +406,16 @@ export class SshChannelMultiplexer { this.highestReceivedSeq = frame.id } - // Process ack from remote: discard timestamps for acked messages - if (frame.ack > this.highestAckedBySelf) { - for (let i = this.highestAckedBySelf + 1; i <= frame.ack; i++) { - this.unackedTimestamps.delete(i) + // Header ACKs are untrusted uint32 values; work stays proportional to the + // bounded set of sequence keys we actually retained. + const acknowledgedSeq = Math.min(frame.ack, this.nextOutgoingSeq - 1) + if (acknowledgedSeq > this.highestAckedBySelf) { + for (const seq of this.unackedTimestamps.keys()) { + if (seq <= acknowledgedSeq) { + this.unackedTimestamps.delete(seq) + } } - this.highestAckedBySelf = frame.ack + this.highestAckedBySelf = acknowledgedSeq } if (frame.type === MessageType.KeepAlive) { @@ -440,7 +487,12 @@ export class SshChannelMultiplexer { Object.defineProperty(err, 'data', { value: msg.error.data }) pending.reject(err) } else { - pending.resolve(msg.result) + try { + pending.beforeResolve?.(msg.result) + pending.resolve(msg.result) + } catch (error) { + pending.reject(error instanceof Error ? error : new Error(String(error))) + } } } @@ -495,15 +547,12 @@ export class SshChannelMultiplexer { // before this tick's fresh probe, then allow the next full window. const resumedAfterWake = sinceLastTick > WAKE_GAP_MS if (resumedAfterWake) { - this.lastReceivedAt = now - for (const seq of this.unackedTimestamps.keys()) { - this.unackedTimestamps.set(seq, now) - } + this.rebaseHealthClocks(now) } this.sendKeepAlive() - if (this.disposed || resumedAfterWake) { + if (this.disposed || resumedAfterWake || this.decoderReadPaused || this.writerSaturated) { return } @@ -529,4 +578,57 @@ export class SshChannelMultiplexer { console.warn(`[ssh-mux] Protocol error: ${err instanceof Error ? err.message : String(err)}`) this.dispose('connection_lost') } + + private trackOutgoingTimestamp(seq: number, liveness: boolean): void { + const limit = liveness ? MAX_UNACKED_TIMESTAMPS : MAX_ORDINARY_UNACKED_TIMESTAMPS + if (this.unackedTimestamps.size < limit) { + this.unackedTimestamps.set(seq, Date.now()) + } + } + + private pauseDecoderReads(): void { + if (this.disposed || this.decoderReadPaused) { + return + } + this.decoderReadPaused = true + try { + this.transport.pauseReads?.() + } catch (error) { + this.handleProtocolError(error) + } + } + + private resumeDecoderReads(): void { + if (!this.decoderReadPaused) { + return + } + this.decoderReadPaused = false + if (this.disposed) { + return + } + this.rebaseHealthClocks(Date.now()) + try { + this.transport.resumeReads?.() + } catch (error) { + this.handleProtocolError(error) + } + } + + private handleWriterSaturationChange(saturated: boolean): void { + this.writerSaturated = saturated + if (!saturated && !this.disposed) { + this.rebaseHealthClocks(Date.now()) + } + } + + private rebaseHealthClocks(now: number): void { + this.lastReceivedAt = now + for (const seq of this.unackedTimestamps.keys()) { + this.unackedTimestamps.set(seq, now) + } + } +} + +function messageLane(msg: JsonRpcMessage): MultiplexerWriterLane { + return 'method' in msg && msg.method === 'pty.data' ? 'ordinary' : 'control' } diff --git a/src/main/ssh/ssh-multiplexer-transport-writer.test.ts b/src/main/ssh/ssh-multiplexer-transport-writer.test.ts new file mode 100644 index 00000000000..ffae9948226 --- /dev/null +++ b/src/main/ssh/ssh-multiplexer-transport-writer.test.ts @@ -0,0 +1,284 @@ +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' +import { + MULTIPLEXER_CONTROL_RESERVE_BYTES, + MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES, + SshMultiplexerTransportWriter, + type MultiplexerTransport, + type MultiplexerWriteSettlement +} from './ssh-multiplexer-transport-writer' + +type WriterHarness = { + transport: MultiplexerTransport + drain: () => void + writes: Buffer[] + callbacks: ((result: MultiplexerWriteSettlement) => void)[] + removeDrain: ReturnType +} + +function transportHarness(writeResults: (boolean | void)[]): WriterHarness { + const emitter = new EventEmitter() + const writes: Buffer[] = [] + const callbacks: ((result: MultiplexerWriteSettlement) => void)[] = [] + const removeDrain = vi.fn() + return { + transport: { + write: (data, onSettled) => { + writes.push(data) + callbacks.push(onSettled!) + return writeResults.shift() + }, + supportsWriteSettlement: true, + onDrain: (callback) => { + emitter.on('drain', callback) + return () => { + removeDrain() + emitter.off('drain', callback) + } + }, + onData: vi.fn(), + onClose: vi.fn() + }, + drain: () => emitter.emit('drain'), + writes, + callbacks, + removeDrain + } +} + +describe('SshMultiplexerTransportWriter', () => { + it('selects queued control before ordinary backlog at the drain boundary', () => { + const harness = transportHarness([false, true, true, true]) + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn()) + + writer.enqueue(Buffer.from('ordinary-1'), 'ordinary') + writer.enqueue(Buffer.from('ordinary-2'), 'ordinary') + writer.enqueue(Buffer.from('ordinary-3'), 'ordinary') + writer.enqueue(Buffer.from('control'), 'control') + + expect(harness.writes.map(String)).toEqual(['ordinary-1']) + harness.drain() + expect(harness.writes.map(String)).toEqual([ + 'ordinary-1', + 'control', + 'ordinary-2', + 'ordinary-3' + ]) + writer.dispose() + }) + + it('preserves FIFO within each lane and prevents ordinary starvation', () => { + const harness = transportHarness([false, ...Array(8).fill(true)]) + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn()) + + writer.enqueue(Buffer.from('ordinary-1'), 'ordinary') + writer.enqueue(Buffer.from('ordinary-2'), 'ordinary') + for (let index = 1; index <= 6; index++) { + writer.enqueue(Buffer.from(`control-${index}`), 'control') + } + + harness.drain() + expect(harness.writes.map(String)).toEqual([ + 'ordinary-1', + 'control-1', + 'control-2', + 'control-3', + 'control-4', + 'ordinary-2', + 'control-5', + 'control-6' + ]) + writer.dispose() + }) + + it('waits for drain and settles each write once', () => { + const harness = transportHarness([false, true, true]) + const failed = vi.fn() + const writer = new SshMultiplexerTransportWriter(harness.transport, failed) + const settlements = [vi.fn(), vi.fn(), vi.fn()] + + writer.enqueue(Buffer.from('ordinary-1'), 'ordinary', settlements[0]) + writer.enqueue(Buffer.from('control'), 'control', settlements[1]) + writer.enqueue(Buffer.from('ordinary-2'), 'ordinary', settlements[2]) + + expect(harness.writes.map(String)).toEqual(['ordinary-1']) + harness.callbacks[0]({ ok: true }) + expect(settlements[0]).toHaveBeenCalledWith({ ok: true }) + expect(harness.writes.map(String)).toEqual(['ordinary-1']) + + harness.drain() + expect(harness.writes.map(String)).toEqual(['ordinary-1', 'control', 'ordinary-2']) + harness.callbacks[1]({ ok: true }) + harness.callbacks[2]({ ok: true }) + expect(settlements.every((settle) => settle.mock.calls.length === 1)).toBe(true) + expect(failed).not.toHaveBeenCalled() + }) + + it('allows one coalesced liveness bypass per saturated write settlement', () => { + const harness = transportHarness([false, true, true, true]) + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn()) + + writer.enqueue(Buffer.from('ordinary-1'), 'ordinary') + writer.enqueue(Buffer.from('ordinary-2'), 'ordinary') + expect(writer.enqueue(Buffer.from('liveness-1'), 'liveness')).toBe(true) + expect(writer.enqueue(Buffer.from('liveness-coalesced'), 'liveness')).toBe(false) + expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1']) + + harness.callbacks[1]({ ok: true }) + expect(writer.enqueue(Buffer.from('liveness-2'), 'liveness')).toBe(true) + expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1', 'liveness-2']) + + harness.drain() + expect(harness.writes.map(String)).toEqual([ + 'ordinary-1', + 'liveness-1', + 'liveness-2', + 'ordinary-2' + ]) + writer.dispose() + }) + + it('reports saturated epochs and their drain boundary exactly once', () => { + const harness = transportHarness([false]) + const saturation = vi.fn() + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn(), saturation) + + writer.enqueue(Buffer.from('ordinary'), 'ordinary') + writer.enqueue(Buffer.from('queued'), 'ordinary') + expect(saturation.mock.calls).toEqual([[true]]) + + harness.drain() + expect(saturation.mock.calls).toEqual([[true], [false]]) + writer.dispose() + }) + + it('keeps a full ordinary lane from consuming the control reserve', () => { + const harness = transportHarness([false, true, true]) + const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn()) + + expect(writer.enqueue(Buffer.alloc(1), 'ordinary')).toBe(true) + expect(writer.enqueue(Buffer.alloc(MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES - 1), 'ordinary')).toBe( + true + ) + expect(writer.enqueue(Buffer.alloc(MULTIPLEXER_CONTROL_RESERVE_BYTES), 'control')).toBe(true) + + expect(harness.writes).toHaveLength(1) + harness.drain() + expect(harness.writes).toHaveLength(3) + writer.dispose() + }) + + it('fails all retained writes once when a callback reports an error', () => { + const harness = transportHarness([false]) + const failed = vi.fn() + const writer = new SshMultiplexerTransportWriter(harness.transport, failed) + const first = vi.fn() + const queued = vi.fn() + const error = new Error('broken pipe') + + writer.enqueue(Buffer.from('first'), 'ordinary', first) + writer.enqueue(Buffer.from('queued'), 'control', queued) + harness.callbacks[0]({ ok: false, error }) + harness.callbacks[0]({ ok: true }) + + expect(first).toHaveBeenCalledOnce() + expect(first).toHaveBeenCalledWith({ ok: false, error }) + expect(queued).toHaveBeenCalledWith({ ok: false, error }) + expect(failed).toHaveBeenCalledWith(error) + expect(harness.removeDrain).toHaveBeenCalledOnce() + }) + + it('closes on queue overflow and rejects the overflowing write', () => { + const harness = transportHarness([false]) + const failed = vi.fn() + const writer = new SshMultiplexerTransportWriter(harness.transport, failed) + const retained = vi.fn() + const overflow = vi.fn() + + writer.enqueue(Buffer.alloc(MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES), 'ordinary', retained) + expect(writer.enqueue(Buffer.alloc(1), 'ordinary', overflow)).toBe(false) + + expect(overflow).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ message: expect.stringContaining('bounded capacity') }) + }) + expect(retained).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ message: expect.stringContaining('bounded capacity') }) + }) + expect(failed).toHaveBeenCalledOnce() + }) + + it('settles legacy callback-less transports on acceptance and drain', () => { + const emitter = new EventEmitter() + const write = vi.fn().mockReturnValueOnce(false).mockReturnValueOnce(undefined) + const transport: MultiplexerTransport = { + write, + onDrain: (callback) => { + emitter.on('drain', callback) + }, + onData: vi.fn(), + onClose: vi.fn() + } + const writer = new SshMultiplexerTransportWriter(transport, vi.fn()) + const first = vi.fn() + const second = vi.fn() + + writer.enqueue(Buffer.from('first'), 'ordinary', first) + writer.enqueue(Buffer.from('second'), 'control', second) + expect(first).not.toHaveBeenCalled() + expect(second).not.toHaveBeenCalled() + + emitter.emit('drain') + expect(first).toHaveBeenCalledWith({ ok: true }) + expect(second).toHaveBeenCalledWith({ ok: true }) + }) + + it('does not miss a drain emitted synchronously by a hostile transport', () => { + let drain = (): void => {} + const write = vi.fn(() => { + drain() + return false + }) + const writer = new SshMultiplexerTransportWriter( + { + write, + onDrain: (callback) => { + drain = callback + }, + onData: vi.fn(), + onClose: vi.fn() + }, + vi.fn() + ) + const first = vi.fn() + const second = vi.fn() + + writer.enqueue(Buffer.from('first'), 'ordinary', first) + writer.enqueue(Buffer.from('second'), 'control', second) + + expect(write).toHaveBeenCalledTimes(2) + expect(first).toHaveBeenCalledWith({ ok: true }) + expect(second).toHaveBeenCalledWith({ ok: true }) + }) + + it('fails deterministically when write(false) has no drain source', () => { + const failed = vi.fn() + const writer = new SshMultiplexerTransportWriter( + { + write: () => false, + onData: vi.fn(), + onClose: vi.fn() + }, + failed + ) + const settled = vi.fn() + + expect(writer.enqueue(Buffer.from('data'), 'ordinary', settled)).toBe(true) + expect(settled).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ message: expect.stringContaining('without drain support') }) + }) + expect(failed).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/ssh/ssh-multiplexer-transport-writer.ts b/src/main/ssh/ssh-multiplexer-transport-writer.ts new file mode 100644 index 00000000000..d428e85889a --- /dev/null +++ b/src/main/ssh/ssh-multiplexer-transport-writer.ts @@ -0,0 +1,270 @@ +import { HEADER_LENGTH, MAX_MESSAGE_SIZE } from './relay-protocol' +import { SshMultiplexerWriterLaneScheduler } from './ssh-multiplexer-writer-lane-scheduler' + +export type MultiplexerWriteSettlement = { ok: true } | { ok: false; error: Error } + +export type MultiplexerTransport = { + write: (data: Buffer, onSettled?: (result: MultiplexerWriteSettlement) => void) => boolean | void + onData: (cb: (data: Buffer) => void) => void + onClose: (cb: () => void) => void + onDrain?: (cb: () => void) => void | (() => void) + supportsWriteSettlement?: boolean + pauseReads?: () => void + resumeReads?: () => void + close?: () => void +} + +export type MultiplexerWriterLane = 'ordinary' | 'control' | 'liveness' + +type WriterEntry = { + data: Buffer + lane: MultiplexerWriterLane + onSettled: (result: MultiplexerWriteSettlement) => void + settled: boolean +} + +export const MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES = 2 * 1024 * 1024 +export const MULTIPLEXER_CONTROL_RESERVE_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH +const ORDINARY_QUEUE_MAX_FRAMES = 2048 +const CONTROL_QUEUE_MAX_FRAMES = 512 + +function onceSettlement( + callback: (result: MultiplexerWriteSettlement) => void +): (result: MultiplexerWriteSettlement) => void { + let settled = false + return (result) => { + if (settled) { + return + } + settled = true + callback(result) + } +} + +export class SshMultiplexerTransportWriter { + private readonly scheduler = new SshMultiplexerWriterLaneScheduler() + private readonly inFlight = new Set() + private readonly settleOnDrain = new Set() + private ordinaryBytes = 0 + private controlBytes = 0 + private ordinaryFrames = 0 + private controlFrames = 0 + private saturated = false + private writing = false + private drainObservedDuringWrite = false + private pumping = false + private closed = false + private livenessOutstanding = false + private removeDrainListener: (() => void) | null = null + + constructor( + private readonly transport: MultiplexerTransport, + private readonly onFailure: (error: Error) => void, + private readonly onSaturationChange: (saturated: boolean) => void = () => {} + ) { + if (transport.onDrain) { + const remove = transport.onDrain(() => this.handleDrain()) + this.removeDrainListener = typeof remove === 'function' ? remove : null + } + } + + enqueue( + data: Buffer, + lane: MultiplexerWriterLane, + onSettled: (result: MultiplexerWriteSettlement) => void = () => {} + ): boolean { + const settle = onceSettlement(onSettled) + if (this.closed) { + settle({ ok: false, error: new Error('Multiplexer writer is closed') }) + return false + } + if (lane === 'liveness' && this.livenessOutstanding) { + return false + } + const admissionError = this.admissionError(data.length, lane) + if (admissionError) { + settle({ ok: false, error: admissionError }) + this.fail(admissionError) + return false + } + const entry = { data, lane, onSettled: settle, settled: false } + this.retain(entry) + if (lane === 'liveness' && this.saturated) { + this.writeEntry(entry) + } else { + this.scheduler.enqueue(entry, lane) + this.pump() + } + return true + } + + dispose(error = new Error('Multiplexer writer disposed')): void { + if (this.closed) { + return + } + this.closed = true + this.saturated = false + this.removeDrainListener?.() + this.removeDrainListener = null + for (const entry of this.scheduler.clear()) { + this.release(entry, { ok: false, error }) + } + for (const entry of Array.from(this.inFlight)) { + this.release(entry, { ok: false, error }) + } + this.settleOnDrain.clear() + } + + private admissionError(bytes: number, lane: MultiplexerWriterLane): Error | null { + const byteLimit = + lane === 'ordinary' ? MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES : MULTIPLEXER_CONTROL_RESERVE_BYTES + const retainedBytes = lane === 'ordinary' ? this.ordinaryBytes : this.controlBytes + const frameLimit = lane === 'ordinary' ? ORDINARY_QUEUE_MAX_FRAMES : CONTROL_QUEUE_MAX_FRAMES + const retainedFrames = lane === 'ordinary' ? this.ordinaryFrames : this.controlFrames + if (retainedBytes + bytes <= byteLimit && retainedFrames < frameLimit) { + return null + } + return new Error(`Multiplexer ${lane} write queue exceeded its bounded capacity`) + } + + private pump(): void { + if (this.pumping || this.closed || this.saturated) { + return + } + this.pumping = true + try { + while (!this.closed && !this.saturated) { + const entry = this.scheduler.select() + if (!entry) { + return + } + this.writeEntry(entry) + } + } finally { + this.pumping = false + } + } + + private writeEntry(entry: WriterEntry): void { + this.inFlight.add(entry) + let callbackResult: MultiplexerWriteSettlement | undefined + let writeReturned = false + const onWriteSettled = (result: MultiplexerWriteSettlement): void => { + if (!writeReturned) { + callbackResult = result + return + } + this.handleWriteSettlement(entry, result) + } + try { + this.writing = true + this.drainObservedDuringWrite = false + const accepted = this.transport.write(entry.data, onWriteSettled) + this.writing = false + writeReturned = true + if (accepted === false) { + if (!this.transport.onDrain) { + throw new Error('Multiplexer transport returned write(false) without drain support') + } + this.setSaturated(!this.drainObservedDuringWrite) + if (this.transport.supportsWriteSettlement !== true && this.saturated) { + this.settleOnDrain.add(entry) + } else if (this.transport.supportsWriteSettlement !== true) { + this.handleWriteSettlement(entry, { ok: true }) + } + } else if (this.transport.supportsWriteSettlement !== true) { + this.handleWriteSettlement(entry, { ok: true }) + } + if (callbackResult) { + this.handleWriteSettlement(entry, callbackResult) + } + } catch (error) { + this.writing = false + writeReturned = true + if (callbackResult) { + this.handleWriteSettlement(entry, callbackResult) + } + this.fail(error instanceof Error ? error : new Error(String(error))) + } + } + + private handleWriteSettlement(entry: WriterEntry, result: MultiplexerWriteSettlement): void { + if (entry.settled) { + return + } + this.release(entry, result) + if (!result.ok) { + this.fail(result.error) + return + } + this.pump() + } + + private handleDrain(): void { + if (this.closed) { + return + } + if (this.writing) { + this.drainObservedDuringWrite = true + return + } + if (!this.saturated) { + return + } + this.setSaturated(false) + for (const entry of Array.from(this.settleOnDrain)) { + this.release(entry, { ok: true }) + } + this.settleOnDrain.clear() + this.pump() + } + + private retain(entry: WriterEntry): void { + if (entry.lane === 'ordinary') { + this.ordinaryBytes += entry.data.length + this.ordinaryFrames++ + } else { + this.controlBytes += entry.data.length + this.controlFrames++ + } + if (entry.lane === 'liveness') { + this.livenessOutstanding = true + } + } + + private release(entry: WriterEntry, result: MultiplexerWriteSettlement): void { + if (entry.settled) { + return + } + entry.settled = true + this.inFlight.delete(entry) + this.settleOnDrain.delete(entry) + if (entry.lane === 'ordinary') { + this.ordinaryBytes -= entry.data.length + this.ordinaryFrames-- + } else { + this.controlBytes -= entry.data.length + this.controlFrames-- + } + if (entry.lane === 'liveness') { + this.livenessOutstanding = false + } + entry.onSettled(result) + } + + private setSaturated(saturated: boolean): void { + if (this.saturated === saturated) { + return + } + this.saturated = saturated + this.onSaturationChange(saturated) + } + + private fail(error: Error): void { + if (this.closed) { + return + } + this.dispose(error) + this.onFailure(error) + } +} diff --git a/src/main/ssh/ssh-multiplexer-writer-lane-scheduler.ts b/src/main/ssh/ssh-multiplexer-writer-lane-scheduler.ts new file mode 100644 index 00000000000..74ad5d593b6 --- /dev/null +++ b/src/main/ssh/ssh-multiplexer-writer-lane-scheduler.ts @@ -0,0 +1,73 @@ +import type { MultiplexerWriterLane } from './ssh-multiplexer-transport-writer' + +const CONTROL_WRITES_BEFORE_ORDINARY = 4 + +type LaneQueue = { + entries: T[] + head: number +} + +function createLaneQueue(): LaneQueue { + return { entries: [], head: 0 } +} + +function hasEntries(queue: LaneQueue): boolean { + return queue.head < queue.entries.length +} + +function shift(queue: LaneQueue): T | undefined { + const entry = queue.entries[queue.head] + if (entry === undefined) { + return undefined + } + queue.head += 1 + if (queue.head === queue.entries.length) { + queue.entries.length = 0 + queue.head = 0 + } + return entry +} + +function clear(queue: LaneQueue): T[] { + const entries = queue.entries.slice(queue.head) + queue.entries.length = 0 + queue.head = 0 + return entries +} + +export class SshMultiplexerWriterLaneScheduler { + private readonly ordinary = createLaneQueue() + private readonly control = createLaneQueue() + private readonly liveness = createLaneQueue() + private controlWritesSinceOrdinary = 0 + + enqueue(entry: T, lane: MultiplexerWriterLane): void { + this[lane].entries.push(entry) + } + + select(): T | undefined { + const liveness = shift(this.liveness) + if (liveness) { + return liveness + } + if ( + hasEntries(this.control) && + (!hasEntries(this.ordinary) || + this.controlWritesSinceOrdinary < CONTROL_WRITES_BEFORE_ORDINARY) + ) { + this.controlWritesSinceOrdinary += 1 + return shift(this.control) + } + const ordinary = shift(this.ordinary) + if (ordinary) { + this.controlWritesSinceOrdinary = 0 + return ordinary + } + return shift(this.control) + } + + clear(): T[] { + this.controlWritesSinceOrdinary = 0 + return [...clear(this.liveness), ...clear(this.control), ...clear(this.ordinary)] + } +} diff --git a/src/main/ssh/ssh-pty-consumer-session.test.ts b/src/main/ssh/ssh-pty-consumer-session.test.ts new file mode 100644 index 00000000000..31e82cf3d5e --- /dev/null +++ b/src/main/ssh/ssh-pty-consumer-session.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it, vi } from 'vitest' +import type { SshChannelMultiplexer } from './ssh-channel-multiplexer' +import { openSshPtyConsumerSession } from './ssh-pty-consumer-session' + +function muxReturning(result: unknown): { + mux: SshChannelMultiplexer + request: ReturnType +} { + const request = vi.fn().mockResolvedValue(result) + return { mux: { request } as unknown as SshChannelMultiplexer, request } +} + +function legacyOwnerGrant(overrides: Record = {}): Record { + return { + protocolVersion: 1, + serverBuildId: 'build-a', + clientGeneration: 3, + role: 'session-owner', + ownerGeneration: 7, + ownerLease: 'lease-a', + ...overrides + } +} + +describe('openSshPtyConsumerSession', () => { + it('makes openClient the one request needed for token-free legacy readiness', async () => { + const { mux, request } = muxReturning(legacyOwnerGrant()) + + await expect( + openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a' + }) + ).resolves.toEqual({ + mode: 'negotiated', + clientInstanceId: 'client-a', + clientGeneration: 3, + ownerGeneration: 7, + ownerLease: 'lease-a' + }) + expect(request).toHaveBeenCalledWith( + 'pty.openClient', + { + protocolVersion: 1, + clientInstanceId: 'client-a', + requestedRole: 'session-owner' + }, + { timeoutMs: 10_000 } + ) + }) + + it('carries recovery generation and lease on reconnect', async () => { + const { mux, request } = muxReturning( + legacyOwnerGrant({ ownerGeneration: 8, ownerLease: 'lease-b' }) + ) + await openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a', + resume: { ownerGeneration: 7, ownerLease: 'lease-a' } + }) + + expect(request.mock.calls[0][1]).toMatchObject({ + resume: { ownerGeneration: 7, ownerLease: 'lease-a' } + }) + }) + + it('rejects a prior or mismatched relay build', async () => { + const { mux } = muxReturning(legacyOwnerGrant({ serverBuildId: 'old-build' })) + + await expect( + openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a' + }) + ).rejects.toThrow('session contract mismatch') + }) + + it('does not silently downgrade when V1 was offered', async () => { + const { mux } = muxReturning(legacyOwnerGrant()) + + await expect( + openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a', + outputFlowControl: { requestedWindowSu: 64 } + }) + ).rejects.toThrow('did not grant') + }) + + it('rejects an unoffered V1 capability in a legacy session', async () => { + const { mux } = muxReturning( + legacyOwnerGrant({ + capabilities: { outputFlowControl: { version: 1, windowSu: 64 } } + }) + ) + + await expect( + openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a' + }) + ).rejects.toThrow('unoffered') + }) + + it('uses explicit token-free fallback only for same-build method-not-found', async () => { + const error = Object.assign(new Error('Method not found: pty.openClient'), { code: -32601 }) + const request = vi.fn().mockRejectedValue(error) + const mux = { request } as unknown as SshChannelMultiplexer + + await expect( + openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a', + allowSameBuildLegacyFallback: true, + outputFlowControl: { requestedWindowSu: 64 } + }) + ).resolves.toEqual({ + mode: 'legacy-fallback', + clientInstanceId: 'client-a', + serverBuildId: 'build-a' + }) + }) + + it.each([ + Object.assign(new Error('timeout'), { code: 'TIMEOUT' }), + Object.assign(new Error('auth failed'), { code: -32000 }), + Object.assign(new Error('method missing'), { code: -32601 }) + ])('does not downgrade an unproved or non-method-not-found error', async (error) => { + const request = vi.fn().mockRejectedValue(error) + const mux = { request } as unknown as SshChannelMultiplexer + + await expect( + openSshPtyConsumerSession(mux, { + clientInstanceId: 'client-a', + expectedServerBuildId: 'build-a', + allowSameBuildLegacyFallback: error.code !== -32601 + }) + ).rejects.toBe(error) + }) +}) diff --git a/src/main/ssh/ssh-pty-consumer-session.ts b/src/main/ssh/ssh-pty-consumer-session.ts new file mode 100644 index 00000000000..3f258f73099 --- /dev/null +++ b/src/main/ssh/ssh-pty-consumer-session.ts @@ -0,0 +1,143 @@ +import { + PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + type PtyConsumerSessionGrant +} from '../../shared/pty-consumer-session' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import type { SshChannelMultiplexer } from './ssh-channel-multiplexer' + +export const SSH_PTY_OPEN_CLIENT_METHOD = 'pty.openClient' +export const SSH_PTY_OPEN_CLIENT_TIMEOUT_MS = 10_000 + +export type SshPtyConsumerOwnerState = { + mode: 'negotiated' + clientInstanceId: string + clientGeneration: number + ownerGeneration: number + ownerLease: string + outputFlowControl?: { + version: 1 + windowSu: number + } +} + +export type SshPtyLegacyFallbackState = { + mode: 'legacy-fallback' + clientInstanceId: string + serverBuildId: string +} + +export type SshPtyConsumerSessionState = SshPtyConsumerOwnerState | SshPtyLegacyFallbackState + +export type OpenSshPtyConsumerSessionOptions = { + clientInstanceId: string + expectedServerBuildId: string | undefined + resume?: Pick + outputFlowControl?: { + requestedWindowSu: number + } + allowSameBuildLegacyFallback?: boolean +} + +function validateGrant( + value: unknown, + options: OpenSshPtyConsumerSessionOptions +): PtyConsumerSessionGrant { + if (typeof value !== 'object' || value === null) { + throw new Error('Remote relay returned an invalid pty.openClient grant') + } + if (!options.expectedServerBuildId) { + throw new Error('Local relay build identity is unavailable') + } + const grant = value as Partial + if ( + grant.protocolVersion !== PTY_CONSUMER_SESSION_PROTOCOL_VERSION || + grant.serverBuildId !== options.expectedServerBuildId + ) { + throw new Error( + `Remote relay session contract mismatch — expected build ${options.expectedServerBuildId}, got ${grant.serverBuildId ?? 'unknown'}` + ) + } + if ( + !Number.isSafeInteger(grant.clientGeneration) || + grant.clientGeneration! <= 0 || + grant.role !== 'session-owner' || + !Number.isSafeInteger(grant.ownerGeneration) || + grant.ownerGeneration! <= 0 || + typeof grant.ownerLease !== 'string' || + grant.ownerLease.length === 0 + ) { + throw new Error('Remote relay did not grant an authenticated PTY session owner') + } + const requestedFlow = options.outputFlowControl + const grantedFlow = grant.capabilities?.outputFlowControl + if (requestedFlow) { + if ( + grantedFlow?.version !== 1 || + !Number.isSafeInteger(grantedFlow.windowSu) || + grantedFlow.windowSu <= 0 || + grantedFlow.windowSu > requestedFlow.requestedWindowSu + ) { + throw new Error('Remote relay did not grant the offered PTY output-flow-control capability') + } + } else if (grantedFlow) { + throw new Error('Remote relay granted an unoffered PTY output-flow-control capability') + } + return grant as PtyConsumerSessionGrant +} + +export async function openSshPtyConsumerSession( + mux: SshChannelMultiplexer, + options: OpenSshPtyConsumerSessionOptions +): Promise { + let result: unknown + try { + result = await mux.request( + SSH_PTY_OPEN_CLIENT_METHOD, + { + protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + clientInstanceId: options.clientInstanceId, + requestedRole: 'session-owner', + ...(options.resume ? { resume: options.resume } : {}), + ...(options.outputFlowControl + ? { + capabilities: { + outputFlowControl: { + versions: [1], + requestedWindowSu: options.outputFlowControl.requestedWindowSu + } + } + } + : {}) + }, + { timeoutMs: SSH_PTY_OPEN_CLIENT_TIMEOUT_MS } + ) + } catch (error) { + const code = (error as { code?: unknown })?.code + if ( + code === -32601 && + options.allowSameBuildLegacyFallback === true && + typeof options.expectedServerBuildId === 'string' && + options.expectedServerBuildId.length > 0 + ) { + return Object.freeze({ + mode: 'legacy-fallback', + clientInstanceId: options.clientInstanceId, + serverBuildId: options.expectedServerBuildId + }) + } + throw error + } + const grant = validateGrant(result, options) + return { + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: grant.clientGeneration, + ownerGeneration: grant.ownerGeneration!, + ownerLease: grant.ownerLease!, + ...(grant.capabilities?.outputFlowControl + ? { outputFlowControl: grant.capabilities.outputFlowControl } + : {}) + } +} + +export const SSH_PTY_SOURCE_WINDOW_SU = DEFAULT_PTY_SOURCE_WINDOW_SU diff --git a/src/main/ssh/ssh-pty-recovery-retention-budget.test.ts b/src/main/ssh/ssh-pty-recovery-retention-budget.test.ts new file mode 100644 index 00000000000..47219e61780 --- /dev/null +++ b/src/main/ssh/ssh-pty-recovery-retention-budget.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget' + +describe('SshPtyRecoveryRetentionBudget', () => { + it('bounds fragmented recovery by per-PTY and session source, charged bytes, and frames', () => { + const budget = new SshPtyRecoveryRetentionBudget({ + perPtySourceSu: 4, + perPtyBytes: 300, + perPtyFrames: 2, + sessionSourceSu: 6, + sessionBytes: 500, + sessionFrames: 3 + }) + + expect(budget.tryRetain('pty-1', 'aa', 2)).toBe(true) + expect(budget.tryRetain('pty-1', 'bb', 2)).toBe(true) + expect(budget.tryRetain('pty-1', 'c', 0)).toBe(false) + expect(budget.tryRetain('pty-2', 'cc', 2)).toBe(true) + expect(budget.snapshot()).toEqual({ sourceSu: 6, bytes: 396, frames: 3, ptys: 2 }) + expect(budget.tryRetain('pty-2', 'd', 1)).toBe(false) + + budget.release('pty-1') + + expect(budget.snapshot()).toEqual({ sourceSu: 2, bytes: 132, frames: 1, ptys: 1 }) + expect(budget.tryRetain('pty-2', 'dd', 2)).toBe(true) + }) + + it('charges UTF-16 storage plus one record before aggregate admission', () => { + const budget = new SshPtyRecoveryRetentionBudget({ + perPtySourceSu: 100, + perPtyBytes: 1_000, + perPtyFrames: 10, + sessionSourceSu: 100, + sessionBytes: 500, + sessionFrames: 10 + }) + + expect(budget.tryRetain('pty-1', '\u0000'.repeat(64), 1)).toBe(true) + expect(budget.snapshot().bytes).toBe(256) + expect(budget.tryRetain('pty-2', '\u0000'.repeat(64), 1)).toBe(false) + expect(budget.snapshot()).toEqual({ sourceSu: 1, bytes: 256, frames: 1, ptys: 1 }) + }) +}) diff --git a/src/main/ssh/ssh-pty-recovery-retention-budget.ts b/src/main/ssh/ssh-pty-recovery-retention-budget.ts new file mode 100644 index 00000000000..873b56a8778 --- /dev/null +++ b/src/main/ssh/ssh-pty-recovery-retention-budget.ts @@ -0,0 +1,95 @@ +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import { chargedPtyRetainedStringBytes } from '../../shared/pty-retained-string-memory' + +export const SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU = DEFAULT_PTY_SOURCE_WINDOW_SU +export const SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES = 2 * 1024 * 1024 +export const SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES = 1_024 +export const SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU = 50 * DEFAULT_PTY_SOURCE_WINDOW_SU +export const SSH_PTY_RECOVERY_SESSION_MAX_BYTES = 64 * 1024 * 1024 +export const SSH_PTY_RECOVERY_SESSION_MAX_FRAMES = 64 * 1_024 + +export type SshPtyRecoveryRetentionLimits = Readonly<{ + perPtySourceSu: number + perPtyBytes: number + perPtyFrames: number + sessionSourceSu: number + sessionBytes: number + sessionFrames: number +}> + +type RetainedPty = { + sourceSu: number + bytes: number + frames: number +} + +const DEFAULT_LIMITS: SshPtyRecoveryRetentionLimits = Object.freeze({ + perPtySourceSu: SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU, + perPtyBytes: SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES, + perPtyFrames: SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES, + sessionSourceSu: SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU, + sessionBytes: SSH_PTY_RECOVERY_SESSION_MAX_BYTES, + sessionFrames: SSH_PTY_RECOVERY_SESSION_MAX_FRAMES +}) + +export class SshPtyRecoveryRetentionBudget { + private readonly retainedByPty = new Map() + private sourceSu = 0 + private bytes = 0 + private frames = 0 + + constructor(private readonly limits: SshPtyRecoveryRetentionLimits = DEFAULT_LIMITS) {} + + tryRetain(ptyId: string, data: string, sourceSu: number): boolean { + if (!Number.isSafeInteger(sourceSu) || sourceSu < 0) { + return false + } + const retained = this.retainedByPty.get(ptyId) ?? { sourceSu: 0, bytes: 0, frames: 0 } + const chargedBytes = chargedPtyRetainedStringBytes(data) + if ( + retained.sourceSu + sourceSu > this.limits.perPtySourceSu || + retained.bytes + chargedBytes > this.limits.perPtyBytes || + retained.frames + 1 > this.limits.perPtyFrames || + this.sourceSu + sourceSu > this.limits.sessionSourceSu || + this.bytes + chargedBytes > this.limits.sessionBytes || + this.frames + 1 > this.limits.sessionFrames + ) { + return false + } + retained.sourceSu += sourceSu + retained.bytes += chargedBytes + retained.frames++ + this.retainedByPty.set(ptyId, retained) + this.sourceSu += sourceSu + this.bytes += chargedBytes + this.frames++ + return true + } + + release(ptyId: string): void { + const retained = this.retainedByPty.get(ptyId) + if (!retained) { + return + } + this.retainedByPty.delete(ptyId) + this.sourceSu -= retained.sourceSu + this.bytes -= retained.bytes + this.frames -= retained.frames + } + + clear(): void { + this.retainedByPty.clear() + this.sourceSu = 0 + this.bytes = 0 + this.frames = 0 + } + + snapshot(): Readonly<{ sourceSu: number; bytes: number; frames: number; ptys: number }> { + return Object.freeze({ + sourceSu: this.sourceSu, + bytes: this.bytes, + frames: this.frames, + ptys: this.retainedByPty.size + }) + } +} diff --git a/src/main/ssh/ssh-pty-retired-source-deliveries.test.ts b/src/main/ssh/ssh-pty-retired-source-deliveries.test.ts new file mode 100644 index 00000000000..377d545592b --- /dev/null +++ b/src/main/ssh/ssh-pty-retired-source-deliveries.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' +import { SshPtyRetiredSourceDeliveries } from './ssh-pty-retired-source-deliveries' + +const source = (deliveryToken: string, relayPtyId = 'pty-1') => ({ + relayPtyId, + deliveryToken, + clientGeneration: 2, + ownerGeneration: 3 +}) + +describe('SshPtyRetiredSourceDeliveries', () => { + it('retains only the latest canceled token for each ordered PTY stream', () => { + const retired = new SshPtyRetiredSourceDeliveries() + + for (let token = 0; token < 10_000; token++) { + retired.retire(1, source(`token-${token}`)) + } + + expect(retired.size).toBe(1) + expect(retired.has(1, source('token-9999'))).toBe(true) + expect(retired.has(1, source('token-9998'))).toBe(false) + }) + + it('retires state at the next activation or PTY exit boundary', () => { + const retired = new SshPtyRetiredSourceDeliveries() + retired.retire(1, source('old-token')) + retired.retire(1, source('other-token', 'pty-2')) + + retired.activate('pty-1') + + expect(retired.has(1, source('old-token'))).toBe(false) + expect(retired.has(1, source('other-token', 'pty-2'))).toBe(true) + expect(retired.size).toBe(1) + }) +}) diff --git a/src/main/ssh/ssh-pty-retired-source-deliveries.ts b/src/main/ssh/ssh-pty-retired-source-deliveries.ts new file mode 100644 index 00000000000..8caca4f889f --- /dev/null +++ b/src/main/ssh/ssh-pty-retired-source-deliveries.ts @@ -0,0 +1,36 @@ +import type { SshPtySourceFrame } from '../providers/ssh-pty-source-frame' + +type SourceDeliveryIdentity = Pick< + SshPtySourceFrame, + 'relayPtyId' | 'deliveryToken' | 'clientGeneration' | 'ownerGeneration' +> + +export class SshPtyRetiredSourceDeliveries { + private readonly keyByRelayPtyId = new Map() + + has(providerGeneration: number, source: SourceDeliveryIdentity): boolean { + return ( + this.keyByRelayPtyId.get(source.relayPtyId) === sourceDeliveryKey(providerGeneration, source) + ) + } + + retire(providerGeneration: number, source: SourceDeliveryIdentity): void { + this.keyByRelayPtyId.set(source.relayPtyId, sourceDeliveryKey(providerGeneration, source)) + } + + activate(relayPtyId: string): void { + this.keyByRelayPtyId.delete(relayPtyId) + } + + clear(): void { + this.keyByRelayPtyId.clear() + } + + get size(): number { + return this.keyByRelayPtyId.size + } +} + +function sourceDeliveryKey(providerGeneration: number, source: SourceDeliveryIdentity): string { + return `${providerGeneration}\0${source.clientGeneration}\0${source.ownerGeneration}\0${source.deliveryToken}` +} diff --git a/src/main/ssh/ssh-relay-cross-version-isolation.test.ts b/src/main/ssh/ssh-relay-cross-version-isolation.test.ts index d566a9b067d..ce5fcecb835 100644 --- a/src/main/ssh/ssh-relay-cross-version-isolation.test.ts +++ b/src/main/ssh/ssh-relay-cross-version-isolation.test.ts @@ -104,7 +104,7 @@ describe('cross-version isolation', () => { // // We feed enough exec results to walk through the deploy: platform, // $HOME, isRelayAlreadyInstalled probe, lock acquire, upload (no exec), - // npm install, finalize, socket probe, socket poll, then GC scan. + // npm install, finalize, socket probe, credential publication, socket poll, then GC scan. const responses: string[] = [ '__ORCA_REMOTE_PLATFORM__ Linux x86_64', // tagged POSIX platform probe '/home/u', // echo $HOME @@ -122,6 +122,7 @@ describe('cross-version isolation', () => { '', // rm -f probe-stderr (best-effort cleanup after probe resolved) '', // touch .install-complete (finalizeInstall) 'DEAD', // launch socket probe + '', // publish the per-launch credential 'READY', // socket poll '', // release .install-lock after relay liveness is observable // GC scan begins here diff --git a/src/main/ssh/ssh-relay-deploy-helpers.test.ts b/src/main/ssh/ssh-relay-deploy-helpers.test.ts index 30e436b979f..acda4594ca2 100644 --- a/src/main/ssh/ssh-relay-deploy-helpers.test.ts +++ b/src/main/ssh/ssh-relay-deploy-helpers.test.ts @@ -9,13 +9,17 @@ import { RELAY_EXIT_CODE_VERSION_MISMATCH } from './ssh-relay-version-mismatch-error' -function createMockChannel(): ClientChannel { +type MockChannel = ClientChannel & { + stdin: EventEmitter & { write: ReturnType } +} + +function createMockChannel(): MockChannel { return Object.assign(new EventEmitter(), { stderr: Object.assign(new EventEmitter(), { resume: vi.fn() }), - stdin: { write: vi.fn() }, + stdin: Object.assign(new EventEmitter(), { write: vi.fn(() => true) }), close: vi.fn(), resume: vi.fn() - }) as unknown as ClientChannel + }) as unknown as MockChannel } // execCommand only rejects with Error; narrow the caught reason (its resolve @@ -204,6 +208,35 @@ describe('waitForSentinel', () => { expect(Buffer.concat(chunks)).toEqual(postSentinelPayload) expect(channel.close).not.toHaveBeenCalled() }) + + it.each(['ssh2 channel', 'system-SSH child stdio'])( + 'forwards write(false), callback settlement, and drain for a %s', + async (shape) => { + const channel = createMockChannel() + if (shape.startsWith('system')) { + Object.assign(channel, { _process: new EventEmitter() }) + } + const callback = vi.fn() + const drain = vi.fn() + channel.stdin.write.mockImplementation((...args: unknown[]) => { + const onWritten = args.find((arg) => typeof arg === 'function') as ( + error?: Error | null + ) => void + onWritten(null) + return false + }) + const transportPromise = waitForSentinel(channel) + channel.emit('data', Buffer.from(RELAY_SENTINEL)) + const transport = await transportPromise + + transport.onDrain?.(drain) + expect(transport.write(Buffer.from('frame'), callback)).toBe(false) + expect(callback).toHaveBeenCalledWith({ ok: true }) + channel.stdin.emit('drain') + expect(drain).toHaveBeenCalledOnce() + expect(transport.supportsWriteSettlement).toBe(true) + } + ) }) describe('execCommand', () => { diff --git a/src/main/ssh/ssh-relay-deploy-helpers.ts b/src/main/ssh/ssh-relay-deploy-helpers.ts index 80d2b78ced2..a035133ddc5 100644 --- a/src/main/ssh/ssh-relay-deploy-helpers.ts +++ b/src/main/ssh/ssh-relay-deploy-helpers.ts @@ -214,7 +214,16 @@ export function waitForSentinel( pendingAfterSentinel = afterSentinel } const transport: MultiplexerTransport = { - write: (buf: Buffer) => channel.stdin.write(buf), + write: (buf: Buffer, onSettled) => { + return channel.stdin.write(buf, (error?: Error | null) => { + onSettled?.(error ? { ok: false, error } : { ok: true }) + }) + }, + supportsWriteSettlement: true, + onDrain: (cb) => { + channel.stdin.on('drain', cb) + return () => channel.stdin.off('drain', cb) + }, onData: (cb) => { dataCallbacks.push(cb) // Why: deliver buffered post-sentinel data to the first @@ -233,6 +242,8 @@ export function waitForSentinel( cb() } }, + pauseReads: () => channel.pause(), + resumeReads: () => channel.resume(), close: () => { channel.close() } diff --git a/src/main/ssh/ssh-relay-deploy.test.ts b/src/main/ssh/ssh-relay-deploy.test.ts index 723cbbbf413..76826a4c670 100644 --- a/src/main/ssh/ssh-relay-deploy.test.ts +++ b/src/main/ssh/ssh-relay-deploy.test.ts @@ -52,6 +52,10 @@ vi.mock('./ssh-remote-node-resolution', () => ({ resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node') })) +vi.mock('./ssh-relay-endpoint-credential', () => ({ + writeRelayEndpointCredential: vi.fn().mockResolvedValue(undefined) +})) + // Why: the versioned-install modules shell out for install state, locking, // and GC. Stub them so deploy tests need no real SSH connection. vi.mock('./ssh-relay-versioned-install', () => ({ @@ -97,13 +101,11 @@ function decodePowerShellCommand(command: string): string | null { return match ? Buffer.from(match[1], 'base64').toString('utf16le') : null } -function extractWindowsSockPath(script: string): string { - return /--sock-path\s+'([^']+)'/.exec(script)?.[1] ?? '' -} +const extractWindowsSockPath = (script: string): string => + /--sock-path\s+'([^']+)'/.exec(script)?.[1] ?? '' -function extractWindowsMarkerPath(script: string): string { - return /-LiteralPath\s+'([^']*\.windows-active-pipe[^']*)'/.exec(script)?.[1] ?? '' -} +const extractWindowsMarkerPath = (script: string): string => + /-LiteralPath\s+'([^']*\.windows-active-pipe[^']*)'/.exec(script)?.[1] ?? '' function makeMockConnection(): SshConnection { return { @@ -115,6 +117,7 @@ function makeMockConnection(): SshConnection { stdout: { on: vi.fn() }, close: vi.fn() }), + writeFile: vi.fn().mockResolvedValue(undefined), sftp: vi.fn().mockResolvedValue({ mkdir: vi.fn((_p: string, cb: (err: Error | null) => void) => cb(null)), createWriteStream: vi.fn().mockReturnValue({ @@ -130,6 +133,10 @@ function makeMockConnection(): SshConnection { } as unknown as SshConnection } +function queueLaunchNamespaceAndDeadSocketProbe(): void { + vi.mocked(execCommand).mockResolvedValueOnce('').mockResolvedValueOnce('DEAD') +} + describe('deployAndLaunchRelay', () => { beforeEach(() => { vi.clearAllMocks() @@ -141,7 +148,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') // tagged POSIX platform probe mockExecCommand.mockResolvedValueOnce('/home/user') // echo $HOME mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll await deployAndLaunchRelay(conn) @@ -159,7 +166,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') mockExecCommand.mockResolvedValueOnce('/home/user') mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll const progress: string[] = [] @@ -179,6 +186,7 @@ describe('deployAndLaunchRelay', () => { .mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') .mockResolvedValueOnce('/home/user') .mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') + .mockResolvedValueOnce('') // launch namespace marker .mockResolvedValueOnce('ALIVE') .mockRejectedValueOnce(unconfirmedCleanup) @@ -195,7 +203,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') mockExecCommand.mockResolvedValueOnce('/home/user') mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') - mockExecCommand.mockResolvedValueOnce('DEAD') + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') await deployAndLaunchRelay(conn) @@ -240,7 +248,7 @@ describe('deployAndLaunchRelay', () => { // Drain the rest of the happy path so a failed assertion does not leave // the deploy promise pending until the overall deploy timeout. mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll releaseRemoteHome('/home/user') deployError = await deployPromise.then( @@ -278,7 +286,7 @@ describe('deployAndLaunchRelay', () => { expect(resolveRemoteNodePath).not.toHaveBeenCalled() mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll releaseRemoteHome('/home/user') await deployPromise @@ -318,7 +326,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockRejectedValueOnce(sessionLimitError) // concurrent node path probe mockExecCommand.mockResolvedValueOnce('/home/user') // sequential fallback $HOME mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll await deployAndLaunchRelay(conn) @@ -348,7 +356,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('/home/user') // concurrent install-state $HOME mockExecCommand.mockResolvedValueOnce('/home/user') // sequential fallback $HOME mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll await deployAndLaunchRelay(conn) @@ -458,20 +466,20 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('/home/user') // sequential fallback $HOME mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe - mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') // socket poll releaseRemoteHome('/home/user') await deployPromise expect(resolveRemoteNodePath).toHaveBeenCalledTimes(2) }) - it('defaults fresh relays to keep-alive-until-reset', async () => { + it('defaults fresh relays to keep-alive-until-reset without rollout artifacts', async () => { const conn = makeMockConnection() const mockExecCommand = vi.mocked(execCommand) mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') mockExecCommand.mockResolvedValueOnce('/home/user') mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') - mockExecCommand.mockResolvedValueOnce('DEAD') + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') await deployAndLaunchRelay(conn) @@ -482,6 +490,8 @@ describe('deployAndLaunchRelay', () => { .find((cmd) => cmd.includes('--detached')) expect(launchCommand).toContain(`--grace-time ${DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS}`) + expect(launchCommand).not.toContain('--pty-source-credit-v1') + expect(launchCommand).not.toContain('.pty-source-credit-policy') }) it('allows an unlimited SSH disconnect grace window', async () => { @@ -490,7 +500,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') mockExecCommand.mockResolvedValueOnce('/home/user') mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') - mockExecCommand.mockResolvedValueOnce('DEAD') + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') await deployAndLaunchRelay(conn, undefined, 0, 'target-a') @@ -501,6 +511,8 @@ describe('deployAndLaunchRelay', () => { .find((cmd) => cmd.includes('--detached')) expect(launchCommand).toContain('--grace-time 0') + expect(launchCommand).not.toContain('--pty-source-credit-v1') + expect(launchCommand).not.toContain('.pty-source-credit-policy') }) it('clamps configured SSH disconnect grace to the seven-day maximum', async () => { @@ -509,7 +521,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') mockExecCommand.mockResolvedValueOnce('/home/user') mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') - mockExecCommand.mockResolvedValueOnce('DEAD') + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') await deployAndLaunchRelay(conn, undefined, MAX_SSH_RELAY_GRACE_PERIOD_SECONDS + 1, 'target-a') @@ -528,7 +540,7 @@ describe('deployAndLaunchRelay', () => { mockExecCommand.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') mockExecCommand.mockResolvedValueOnce('/home/user') mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') - mockExecCommand.mockResolvedValueOnce('DEAD') + queueLaunchNamespaceAndDeadSocketProbe() mockExecCommand.mockResolvedValueOnce('READY') await deployAndLaunchRelay(conn) @@ -659,6 +671,7 @@ describe('deployAndLaunchRelay', () => { setTimeout(() => resolve('ORCA-NATIVE-DEPS-OK'), 899_900) ) ) + .mockResolvedValueOnce('') // launch namespace marker .mockResolvedValueOnce('DEAD') .mockImplementationOnce((_conn, _command, options) => { return new Promise((_resolve, reject) => { @@ -685,9 +698,9 @@ describe('deployAndLaunchRelay', () => { expect(result).toBeInstanceOf(Error) expect((result as Error).message).toBe('Relay deployment timed out after 900s') expect(launchChannel.close).toHaveBeenCalledTimes(1) - expect(mockExecCommand).toHaveBeenCalledTimes(5) + expect(mockExecCommand).toHaveBeenCalledTimes(6) await vi.advanceTimersByTimeAsync(10_000) - expect(mockExecCommand).toHaveBeenCalledTimes(5) + expect(mockExecCommand).toHaveBeenCalledTimes(6) } finally { vi.useRealTimers() } @@ -701,11 +714,13 @@ describe('deployAndLaunchRelay', () => { .mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') // tagged POSIX platform probe A .mockResolvedValueOnce('/home/user') // $HOME A .mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe A + .mockResolvedValueOnce('') // launch namespace marker A .mockResolvedValueOnce('DEAD') // probe A .mockResolvedValueOnce('READY') // poll A .mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64') // tagged POSIX platform probe B .mockResolvedValueOnce('/home/user') // $HOME B .mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe B + .mockResolvedValueOnce('') // launch namespace marker B .mockResolvedValueOnce('DEAD') // probe B .mockResolvedValueOnce('READY') // poll B @@ -766,6 +781,8 @@ describe('deployAndLaunchRelay', () => { '"C:/Users/me user/.orca-remote/relay-0.1.0+abcdef012345/agent-hooks/orca-relay-' ) expect(launchScript).toContain('--endpoint-dir') + expect(launchScript).not.toContain('--pty-source-credit-v1') + expect(launchScript).not.toContain('.pty-source-credit-policy') expect(launchScript).not.toContain('\\\\.\\pipe\\agent-hooks') const waitScript = decodedScripts.find((script) => script.includes('deadline=Date.now()')) ?? '' expect(waitScript).toContain('setTimeout(attempt,intervalMs)') diff --git a/src/main/ssh/ssh-relay-deploy.ts b/src/main/ssh/ssh-relay-deploy.ts index 81165efc4d0..d57e4ad55a4 100644 --- a/src/main/ssh/ssh-relay-deploy.ts +++ b/src/main/ssh/ssh-relay-deploy.ts @@ -11,6 +11,7 @@ import { isUnconfirmedSshCommandTermination } from './ssh-relay-deploy-helpers' import { uploadRelayDirectory, writeRelayFile } from './ssh-relay-install-transfers' +import { writeRelayEndpointCredential } from './ssh-relay-endpoint-credential' import { createRelayInstallMarkerCommand, createRelayInstallNamespace, @@ -75,12 +76,14 @@ import { export type RelayDeployResult = { transport: MultiplexerTransport + serverBuildId?: string platform: RelayPlatform hostPlatform?: RemoteHostPlatform remoteHome?: string remoteRelayDir?: string nodePath?: string sockPath?: string + credentialFile?: string } class RelayDirectoryGcConflictError extends Error { @@ -324,6 +327,7 @@ async function deployAndLaunchRelayAttempt( let ownsInstallLock = false let launchGcClaimToken: string | undefined + let launchNamespace: RelayInstallNamespace | undefined if (alreadyInstalled) { const launchFence = await repairInstalledNativeDeps( conn, @@ -336,6 +340,7 @@ async function deployAndLaunchRelayAttempt( ) ownsInstallLock = launchFence.ownsInstallLock launchGcClaimToken = launchFence.gcClaimToken + launchNamespace = launchFence.sftpNamespace deploySignal?.throwIfAborted() } else { // Why: serialize concurrent first-installs via a host-native exclusive lock; the loser polls to re-check installed or steal a stale lock. @@ -348,7 +353,7 @@ async function deployAndLaunchRelayAttempt( signal: deploySignal })) ) { - const installNamespace = createInstallNamespaceIfSupported( + launchNamespace = createInstallNamespaceIfSupported( conn, hostPlatform, homeRelativeRelayDir @@ -363,7 +368,7 @@ async function deployAndLaunchRelayAttempt( fullVersion, hostPlatform, deploySignal, - installNamespace + launchNamespace ) console.log('[ssh-relay] Upload complete') @@ -377,7 +382,7 @@ async function deployAndLaunchRelayAttempt( nodePath, deploySignal, [], - installNamespace + launchNamespace ) console.log('[ssh-relay] Native deps installed') @@ -434,12 +439,14 @@ async function deployAndLaunchRelayAttempt( return { transport: launched.transport, + serverBuildId: fullVersion, platform, hostPlatform, remoteHome, remoteRelayDir, nodePath: launched.nodePath, - sockPath: launched.sockPath + sockPath: launched.sockPath, + credentialFile: launched.credentialFile } } @@ -592,7 +599,11 @@ async function repairInstalledNativeDeps( nodePath: string, homeRelativeRelayDir: string, signal?: AbortSignal -): Promise<{ ownsInstallLock: boolean; gcClaimToken?: string }> { +): Promise<{ + ownsInstallLock: boolean + gcClaimToken?: string + sftpNamespace?: RelayInstallNamespace +}> { const initialProbe = await probeRequiredNativeDeps( conn, remoteDir, @@ -627,7 +638,27 @@ async function repairInstalledNativeDeps( : undefined if (initialProbe.available) { // Why: even a healthy reconnect stays fenced until launch liveness is observable, or cross-version GC can rename after this probe. - return { ownsInstallLock: lockResult === 'acquired', gcClaimToken } + if (lockResult !== 'acquired') { + return { ownsInstallLock: false, gcClaimToken } + } + try { + return { + ownsInstallLock: true, + sftpNamespace: await createRelayLaunchNamespace( + conn, + hostPlatform, + remoteDir, + homeRelativeRelayDir, + signal + ) + } + } catch (err) { + signal?.throwIfAborted() + console.warn( + `[ssh-relay] Launch namespace marker is unconfirmed at ${remoteDir}; deferring lock ownership to stale recovery` + ) + return { ownsInstallLock: !isUnconfirmedSshCommandTermination(err) } + } } // Why: an already-installed relay can launch degraded, so native-deps repair is best-effort — lock contention and failures must not abort the connection. @@ -641,9 +672,10 @@ async function repairInstalledNativeDeps( try { // Why: older complete relay dirs predate @parcel/watcher; re-probe under the lock so only one reconnect mutates the dir. const probe = await probeRequiredNativeDeps(conn, remoteDir, hostPlatform, nodePath, signal) + let repairNamespace: RelayInstallNamespace | undefined if (!probe.available) { // Why: only stamp ownership once the locked recheck proves this connection is the one about to write. - const repairNamespace = await createRepairInstallMarker( + repairNamespace = await createRelayLaunchNamespace( conn, hostPlatform, remoteDir, @@ -662,7 +694,7 @@ async function repairInstalledNativeDeps( ) await finalizeInstall(conn, remoteDir, hostPlatform, { signal, releaseLock: false }) } - return { ownsInstallLock: true } + return { ownsInstallLock: true, sftpNamespace: repairNamespace } } catch (err) { const terminationUnconfirmed = isUnconfirmedSshCommandTermination(err) // Why: hold a confirmed-failure lock through degraded launch so GC can't move the relay before liveness is visible. @@ -677,10 +709,10 @@ async function repairInstalledNativeDeps( } /** - * Stamp this connection as the install owner during repair. Marker creation is - * best-effort: without it the writes simply keep using the shell path. + * Stamp this connection as the launch writer while it owns the install lock. + * Confirmed marker failures fall back to shell-path credential generation. */ -async function createRepairInstallMarker( +async function createRelayLaunchNamespace( conn: SshConnection, hostPlatform: RemoteHostPlatform, remoteDir: string, @@ -1157,7 +1189,12 @@ async function launchRelay( graceTimeSeconds?: number, relayInstanceId?: string, signal?: AbortSignal -): Promise<{ transport: MultiplexerTransport; nodePath: string; sockPath: string }> { +): Promise<{ + transport: MultiplexerTransport + nodePath: string + sockPath: string + credentialFile: string +}> { // Why: graceTimeSeconds comes from user-editable SshTarget config; floor+clamp to an integer prevents shell injection if the type ever loosened. const requestedGraceTime = Math.floor(graceTimeSeconds ?? DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS) const graceTime = @@ -1173,21 +1210,23 @@ async function launchRelay( const sockName = relaySocketNameForInstanceId(relayInstanceId) const sockFile = relayEndpointForHost(hostPlatform, remoteDir, sockName) const endpointDir = relayHookEndpointDirForHost(hostPlatform, remoteDir, sockFile) + const credentialFile = joinRemotePath(hostPlatform, remoteDir, `${sockName}.credential`) if (isWindowsRemoteHost(hostPlatform)) { const activePipeMarkerPath = windowsActivePipeMarkerPath(hostPlatform, remoteDir, sockName) - const activeEndpoint = (await readWindowsActiveRelayEndpoint( + const discoveredActiveEndpoint = await readWindowsActiveRelayEndpoint( conn, hostPlatform, remoteDir, activePipeMarkerPath, signal - )) ?? { + ) + const activeEndpoint = discoveredActiveEndpoint ?? { sockPath: sockFile, endpointDir } const fallbackEndpoint = buildWindowsRelayFallbackEndpoint(hostPlatform, remoteDir, sockName) - return launchWindowsRelay( + const launched = await launchWindowsRelay( conn, hostPlatform, { @@ -1197,10 +1236,12 @@ async function launchRelay( endpointDir: activeEndpoint.endpointDir, graceTime, activePipeMarkerPath, - reconnectFallback: fallbackEndpoint + reconnectFallback: fallbackEndpoint, + credentialFile }, signal ) + return { ...launched, credentialFile } } // Why: after a restart the relay may still be alive in its grace period; --connect to its socket preserves PTY state and scrollback. @@ -1215,12 +1256,12 @@ async function launchRelay( console.log('[ssh-relay] Existing relay socket found, attempting reconnect...') try { const channel = await conn.exec( - `cd ${escapedDir} && ${escapedNode} relay.js --connect --sock-path ${shellEscape(sockFile)}`, + `cd ${escapedDir} && ${escapedNode} relay.js --connect --sock-path ${shellEscape(sockFile)} --credential-file ${shellEscape(credentialFile)}`, { signal } ) const transport = await waitForSentinel(channel, signal) console.log('[ssh-relay] Reconnected to existing relay via socket') - return { transport, nodePath, sockPath: sockFile } + return { transport, nodePath, sockPath: sockFile, credentialFile } } catch (err) { signal?.throwIfAborted() console.warn( @@ -1249,8 +1290,11 @@ async function launchRelay( // Why: relay must outlive the SSH connection so PTY sessions survive app restarts — nohup + ${shellEscape(logFile)} 2>&1 ${shellEscape(logFile)} 2>&1 {}) launchChannel.on('error', () => {}) @@ -1300,10 +1344,15 @@ async function launchRelay( // Why: backgrounded relay's stdout goes to a log file, not the exec channel; --connect bridges this channel to its Unix socket. const channel = await conn.exec( - `cd ${escapedDir} && ${escapedNode} relay.js --connect --sock-path ${shellEscape(sockFile)}`, + `cd ${escapedDir} && ${escapedNode} relay.js --connect --sock-path ${shellEscape(sockFile)} --credential-file ${shellEscape(credentialFile)}`, { signal } ) - return { transport: await waitForSentinel(channel, signal), nodePath, sockPath: sockFile } + return { + transport: await waitForSentinel(channel, signal), + nodePath, + sockPath: sockFile, + credentialFile + } } function waitForRelayPoll(delayMs: number, signal?: AbortSignal): Promise { @@ -1398,6 +1447,7 @@ type WindowsRelayLaunchOptions = { nodePath: string graceTime: number activePipeMarkerPath: string + credentialFile: string } & WindowsRelayEndpoint & { reconnectFallback?: WindowsRelayEndpoint } @@ -1467,6 +1517,13 @@ async function launchWindowsRelay( const logFile = joinRemotePath(hostPlatform, launchOpts.remoteDir, 'relay.log') const errFile = joinRemotePath(hostPlatform, launchOpts.remoteDir, 'relay.err.log') + await writeRelayEndpointCredential( + conn, + hostPlatform, + launchOpts.nodePath, + launchOpts.credentialFile, + { signal } + ) await execHostCommand( conn, hostPlatform, @@ -1478,7 +1535,8 @@ async function launchWindowsRelay( launchOpts.endpointDir, launchOpts.graceTime, logFile, - errFile + errFile, + launchOpts.credentialFile ), { signal } ) @@ -1529,11 +1587,18 @@ async function connectWindowsRelay( remoteDir: string nodePath: string sockPath: string + credentialFile: string }, signal?: AbortSignal ): Promise { const channel = await conn.exec( - windowsRelayConnectCommand(hostPlatform, opts.nodePath, opts.remoteDir, opts.sockPath), + windowsRelayConnectCommand( + hostPlatform, + opts.nodePath, + opts.remoteDir, + opts.sockPath, + opts.credentialFile + ), { wrapCommand: false, signal } ) return waitForSentinel(channel, signal) @@ -1543,13 +1608,14 @@ function windowsRelayConnectCommand( hostPlatform: RemoteHostPlatform, nodePath: string, remoteDir: string, - sockPath: string + sockPath: string, + credentialFile: string ): string { return commandWithNodePath( hostPlatform, nodePath, remoteDir, - `& ${powerShellLiteral(nodePath)} relay.js --connect --sock-path ${powerShellLiteral(sockPath)}` + `& ${powerShellLiteral(nodePath)} relay.js --connect --sock-path ${powerShellLiteral(sockPath)} --credential-file ${powerShellLiteral(credentialFile)}` ) } @@ -1561,7 +1627,8 @@ function windowsRelayLaunchCommand( endpointDir: string, graceTime: number, logFile: string, - errFile: string + errFile: string, + credentialFile: string ): string { const relayScript = joinRemotePath(hostPlatform, remoteDir, 'relay.js') // Why: Windows sshd kills the exec channel's process tree on close; WMI re-parents the detached relay to survive. @@ -1574,6 +1641,8 @@ function windowsRelayLaunchCommand( String(graceTime), '--sock-path', quoted(sockPath), + '--credential-file', + quoted(credentialFile), '--endpoint-dir', quoted(endpointDir), // Why: --log-file owns rotation; shell redirects still capture pre-JS boot/crash output. @@ -1588,6 +1657,7 @@ function windowsRelayLaunchCommand( nodePath, remoteDir, [ + `& icacls.exe ${powerShellLiteral(credentialFile)} /inheritance:r /grant:r "$($env:USERNAME):(R,W)" | Out-Null`, `$result = Invoke-CimMethod -ClassName Win32_Process -MethodName Create -Arguments @{ CommandLine = ${powerShellLiteral(wmiCommandLine)}; CurrentDirectory = ${powerShellLiteral(remoteDir)} }`, `if ($result.ReturnValue -ne 0) { throw "Win32_Process.Create failed with $($result.ReturnValue)" }` ].join('; ') diff --git a/src/main/ssh/ssh-relay-endpoint-credential.test.ts b/src/main/ssh/ssh-relay-endpoint-credential.test.ts new file mode 100644 index 00000000000..3520b94c8e8 --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-credential.test.ts @@ -0,0 +1,50 @@ +import { Buffer } from 'node:buffer' +import { describe, expect, it } from 'vitest' +import { relayEndpointCredentialWriteCommand } from './ssh-relay-endpoint-credential' +import { getRemoteHostPlatform } from './ssh-remote-platform' + +function decodePowerShellCommand(command: string): string { + const encoded = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/)?.[1] + if (!encoded) { + throw new Error(`Expected an encoded PowerShell command: ${command}`) + } + return Buffer.from(encoded, 'base64').toString('utf16le') +} + +describe('relay endpoint credential writes', () => { + it('publishes a POSIX credential only from a restrictive temporary file', () => { + const command = relayEndpointCredentialWriteCommand( + getRemoteHostPlatform('linux-x64'), + '/opt/orca node/bin/node', + '/home/me user/.orca-remote/relay.sock.credential' + ) + + expect(command).toContain('{flag:"wx",mode:0o600}') + expect(command).toContain('fs.writeFileSync(t,') + expect(command).not.toContain('fs.writeFileSync(p,') + expect(command.indexOf('fs.writeFileSync(t,')).toBeLessThan( + command.indexOf('fs.renameSync(t,p)') + ) + expect(command).toContain("'/home/me user/.orca-remote/relay.sock.credential'") + }) + + it('creates a Windows credential with its owner-only ACL before publication', () => { + const script = decodePowerShellCommand( + relayEndpointCredentialWriteCommand( + getRemoteHostPlatform('win32-x64'), + 'C:/Program Files/nodejs/node.exe', + 'C:/Users/me user/.orca-remote/relay.sock.credential' + ) + ) + + expect(script).toContain("$path = 'C:/Users/me user/.orca-remote/relay.sock.credential'") + expect(script).toContain('$security.SetAccessRuleProtection($true,$false)') + expect(script).toContain('[System.IO.FileStream]::new($tempPath') + expect(script).toContain('[System.IO.FileOptions]::WriteThrough,$security)') + expect(script.indexOf('[System.IO.FileStream]::new')).toBeLessThan( + script.indexOf('[System.IO.File]::Move($tempPath,$path)') + ) + expect(script).not.toContain('Set-Acl') + expect(script).not.toContain('icacls') + }) +}) diff --git a/src/main/ssh/ssh-relay-endpoint-credential.ts b/src/main/ssh/ssh-relay-endpoint-credential.ts new file mode 100644 index 00000000000..896aec31211 --- /dev/null +++ b/src/main/ssh/ssh-relay-endpoint-credential.ts @@ -0,0 +1,61 @@ +import type { SshConnection } from './ssh-connection' +import { shellEscape } from './ssh-connection-utils' +import { execCommand } from './ssh-relay-deploy-helpers' +import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform' +import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell' + +const POSIX_CREDENTIAL_SCRIPT = + 'const fs=require("fs"),crypto=require("crypto"),p=process.argv[1],' + + 't=p+"."+process.pid+"."+crypto.randomBytes(8).toString("hex")+".tmp";' + + 'try{' + + 'fs.writeFileSync(t,crypto.randomBytes(32).toString("base64url"),{flag:"wx",mode:0o600});' + + 'fs.renameSync(t,p)' + + '}finally{try{fs.unlinkSync(t)}catch(e){if(e.code!=="ENOENT")throw e}}' + +export function relayEndpointCredentialWriteCommand( + hostPlatform: RemoteHostPlatform, + nodePath: string, + credentialFile: string +): string { + if (!isWindowsRemoteHost(hostPlatform)) { + return `${shellEscape(nodePath)} -e ${shellEscape(POSIX_CREDENTIAL_SCRIPT)} ${shellEscape(credentialFile)}` + } + return powerShellCommand( + [ + '$ErrorActionPreference = "Stop"', + `$path = ${powerShellLiteral(credentialFile)}`, + '$tempPath = $path + "." + [Guid]::NewGuid().ToString("N") + ".tmp"', + '$identity = [System.Security.Principal.WindowsIdentity]::GetCurrent().User', + '$security = [System.Security.AccessControl.FileSecurity]::new()', + '$security.SetOwner($identity)', + '$security.SetAccessRuleProtection($true,$false)', + '$rule = [System.Security.AccessControl.FileSystemAccessRule]::new($identity,[System.Security.AccessControl.FileSystemRights]::FullControl,[System.Security.AccessControl.AccessControlType]::Allow)', + '$security.AddAccessRule($rule)', + '$random = [byte[]]::new(32)', + '$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()', + 'try { $rng.GetBytes($random) } finally { $rng.Dispose() }', + '$credential = [Convert]::ToBase64String($random).TrimEnd("=").Replace("+","-").Replace("/","_")', + '$data = [System.Text.UTF8Encoding]::new($false).GetBytes($credential)', + '$stream = [System.IO.FileStream]::new($tempPath,[System.IO.FileMode]::CreateNew,[System.Security.AccessControl.FileSystemRights]::Write,[System.IO.FileShare]::None,4096,[System.IO.FileOptions]::WriteThrough,$security)', + 'try { $stream.Write($data,0,$data.Length) } finally { $stream.Dispose() }', + 'try { [System.IO.File]::Delete($path); [System.IO.File]::Move($tempPath,$path) } finally { [System.IO.File]::Delete($tempPath) }' + ].join('; ') + ) +} + +export async function writeRelayEndpointCredential( + conn: SshConnection, + hostPlatform: RemoteHostPlatform, + nodePath: string, + credentialFile: string, + options?: { signal?: AbortSignal } +): Promise { + await execCommand( + conn, + relayEndpointCredentialWriteCommand(hostPlatform, nodePath, credentialFile), + { + wrapCommand: !isWindowsRemoteHost(hostPlatform), + signal: options?.signal + } + ) +} diff --git a/src/main/ssh/ssh-relay-gc-retry.test.ts b/src/main/ssh/ssh-relay-gc-retry.test.ts index 76a5cb18444..ae70ff68fa6 100644 --- a/src/main/ssh/ssh-relay-gc-retry.test.ts +++ b/src/main/ssh/ssh-relay-gc-retry.test.ts @@ -69,7 +69,8 @@ function makeConnection(): SshConnection { } return { canRunConcurrentExecCommands: vi.fn().mockReturnValue(true), - exec: vi.fn().mockResolvedValue(channel) + exec: vi.fn().mockResolvedValue(channel), + writeFile: vi.fn().mockResolvedValue(undefined) } as unknown as SshConnection } diff --git a/src/main/ssh/ssh-relay-native-deps-install-fixture.ts b/src/main/ssh/ssh-relay-native-deps-install-fixture.ts index ea2b1aa073b..f2f9ebf4298 100644 --- a/src/main/ssh/ssh-relay-native-deps-install-fixture.ts +++ b/src/main/ssh/ssh-relay-native-deps-install-fixture.ts @@ -71,6 +71,7 @@ export function makeRepairToolchainSkipExecResponses(): ExecResponse[] { '', // cat probe stderr '', // rm -f probe stderr 'DEAD', + '', // publish the per-launch credential 'READY' ] } @@ -80,7 +81,7 @@ export function decodePowerShellCommand(command: string): string | null { return match ? Buffer.from(match[1], 'base64').toString('utf16le') : null } -// Happy-path exec order: uname, $HOME, mkdir, chmod node, npm install, chmod prebuilds, probe, [cat stderr + rm if MISSING], [rebuild → chmod → re-probe if MISSING], DEAD, READY. +// Happy-path exec order ends with socket probe, credential publication, then readiness poll. // When the probe rejects (SSH channel close or vanished install dir), the catch skips both stderr-capture and the rm. // A failed npm install takes one of the two early branches below instead, which never reach `probe`. export function makeExecResponses(opts: { @@ -134,6 +135,7 @@ export function makeExecResponses(opts: { '', // cat probe stderr '', // rm -f probe stderr 'DEAD', + '', // publish the per-launch credential 'READY' ] } @@ -176,6 +178,6 @@ export function makeExecResponses(opts: { slots.push('') // rm -f stderr after rebuild probe } } - slots.push('DEAD', 'READY') + slots.push('DEAD', '', 'READY') return slots } diff --git a/src/main/ssh/ssh-relay-native-deps-install.test.ts b/src/main/ssh/ssh-relay-native-deps-install.test.ts index 2d98dfa95e0..3036d1d633c 100644 --- a/src/main/ssh/ssh-relay-native-deps-install.test.ts +++ b/src/main/ssh/ssh-relay-native-deps-install.test.ts @@ -567,6 +567,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { 'MISSING\n', // rebuilt native process still cannot load '', // no persisted active pipe marker 'WAITING', + '', // publish the per-launch credential '', // WMI relay launch 'READY', '' // persist active pipe marker @@ -661,6 +662,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { 'ORCA-NPTY-PROBE-OK\n', '', // rm probe stderr 'DEAD', + '', // publish the per-launch credential 'READY' ]) @@ -712,6 +714,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { '', // SFTP-namespace install-owner marker (repair) { reject: 'npm ERR! network ETIMEDOUT' }, // npm install fails (offline) 'DEAD', + '', // remote credential generation after degraded repair 'READY' ]) @@ -739,6 +742,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { }) ) .mockResolvedValueOnce('DEAD') + .mockResolvedValueOnce('') // remote credential generation after degraded repair .mockResolvedValueOnce('READY') await deployAndLaunchRelay(conn) @@ -880,7 +884,14 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true) vi.mocked(tryAcquireRelayRepairLock).mockResolvedValueOnce(lockResult) const conn = makeMockConnection(sftpCapture) - feed(['__ORCA_REMOTE_PLATFORM__ Linux x86_64', '/home/u', 'MISSING', 'DEAD', 'READY']) + feed([ + '__ORCA_REMOTE_PLATFORM__ Linux x86_64', + '/home/u', + 'MISSING', + 'DEAD', + '', // remote credential generation without a namespace marker + 'READY' + ]) await deployAndLaunchRelay(conn) @@ -899,7 +910,9 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { '__ORCA_REMOTE_PLATFORM__ Linux x86_64', '/home/u', 'ORCA-NATIVE-DEPS-OK', + '', // launch namespace marker 'DEAD', + '', // publish the per-launch credential 'READY' ]) @@ -922,7 +935,9 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => { '__ORCA_REMOTE_PLATFORM__ Linux x86_64', '/home/u', 'ORCA-NATIVE-DEPS-OK', + '', // launch namespace marker 'DEAD', + '', // publish the per-launch credential 'READY' ]) diff --git a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts index d6c98bf03a0..bf0a1891cb4 100644 --- a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts +++ b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts @@ -15,6 +15,7 @@ import { import { agentHookServer, _internals as agentHookInternals } from '../agent-hooks/server' import { getSshPtyProvider } from '../ipc/pty' import { toAppSshPtyId } from '../providers/ssh-pty-id' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({ getCohortAtEmitMock: vi.fn(), @@ -97,6 +98,21 @@ function createFakeRelay(): FakeRelay { }) relayFeed = (data) => dispatcher.feed(data) + dispatcher.onRequest('pty.openClient', async (params) => ({ + protocolVersion: 1, + serverBuildId: 'test-relay-build', + clientGeneration: 1, + role: 'session-owner', + ownerGeneration: + typeof (params.resume as { ownerGeneration?: unknown } | undefined)?.ownerGeneration === + 'number' + ? (params.resume as { ownerGeneration: number }).ownerGeneration + 1 + : 1, + ownerLease: 'test-owner-lease', + capabilities: { + outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } + } + })) dispatcher.onRequest('session.resolveHome', async (params) => ({ resolvedPath: params.path === '~' ? '/home/orca' : params.path })) @@ -224,6 +240,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) const events: CapturedStatus[] = [] @@ -277,8 +294,16 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { const initialRelay = createFakeRelay() relay = createFakeRelay() vi.mocked(deployAndLaunchRelay) - .mockResolvedValueOnce({ transport: initialRelay.transport, platform: 'linux-x64' }) - .mockResolvedValueOnce({ transport: relay.transport, platform: 'linux-x64' }) + .mockResolvedValueOnce({ + transport: initialRelay.transport, + serverBuildId: 'test-relay-build', + platform: 'linux-x64' + }) + .mockResolvedValueOnce({ + transport: relay.transport, + serverBuildId: 'test-relay-build', + platform: 'linux-x64' + }) const clearListener = vi.fn() agentHookServer.setPaneStatusClearListener(clearListener) session = createSession('conn-clear') @@ -318,6 +343,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { ) vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) const events: CapturedStatus[] = [] @@ -345,6 +371,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) const events: CapturedStatus[] = [] @@ -375,6 +402,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) @@ -434,6 +462,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) const ingestSpy = vi.spyOn(agentHookServer, 'ingestRemote') @@ -488,6 +517,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) @@ -535,6 +565,7 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ transport: relay.transport, + serverBuildId: 'test-relay-build', platform: 'linux-x64' }) diff --git a/src/main/ssh/ssh-relay-session-data-delivery.test.ts b/src/main/ssh/ssh-relay-session-data-delivery.test.ts index 901f7940d31..61c7fe86198 100644 --- a/src/main/ssh/ssh-relay-session-data-delivery.test.ts +++ b/src/main/ssh/ssh-relay-session-data-delivery.test.ts @@ -1,19 +1,65 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { SshRelaySession } from './ssh-relay-session' -import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' +import { + createMismatchedOwnerRecoveryError, + createMockDeps, + mockDeploySuccess +} from './ssh-relay-session-test-fixtures' -const { muxRequestMock } = vi.hoisted(() => ({ muxRequestMock: vi.fn() })) +const { + acceptOutputDataMock, + muxRequestMock, + onNotificationByMethodMock, + notifyWithSettlementMock, + openConsumerSessionMock, + pauseAdapterMock, + muxDisposeMock, + attachForReconnectMock, + beginMigrationMock, + ptyDataHandlerRef +} = vi.hoisted(() => ({ + acceptOutputDataMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn(), + onNotificationByMethodMock: vi.fn(), + notifyWithSettlementMock: vi.fn(), + openConsumerSessionMock: vi.fn(), + pauseAdapterMock: vi.fn(), + muxDisposeMock: vi.fn(), + attachForReconnectMock: vi.fn().mockResolvedValue({}), + beginMigrationMock: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + ptyDataHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) } +})) vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: openConsumerSessionMock +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: acceptOutputDataMock, + acceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined), + allocateSshPtyProviderGeneration: vi.fn(() => 23), + beginSshPtyOutputGenerationMigration: beginMigrationMock, + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}), + applySshPtySourceCancellationProof: vi.fn(), + applySshPtySourceRecoveryCancellationProof: vi.fn() +})) vi.mock('./ssh-channel-multiplexer', () => ({ SshChannelMultiplexer: class MockSshChannelMultiplexer { notify = vi.fn() + notifyWithSettlement = notifyWithSettlementMock request = muxRequestMock onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = onNotificationByMethodMock.mockImplementation(() => () => {}) onRequest = vi.fn().mockReturnValue(() => {}) onDispose = vi.fn().mockReturnValue(() => {}) - dispose = vi.fn() + dispose = muxDisposeMock isDisposed = vi.fn().mockReturnValue(false) } })) @@ -26,11 +72,15 @@ vi.mock('../providers/ssh-pty-provider', () => ({ isSshPtyNotFoundError: vi.fn().mockReturnValue(false), isSshPtyIdentityMismatchError: vi.fn().mockReturnValue(false), SshPtyProvider: class MockSshPtyProvider { - onData = vi.fn().mockReturnValue(() => {}) + onData = vi.fn().mockImplementation((handler) => { + ptyDataHandlerRef.current = handler + return () => {} + }) onReplay = vi.fn().mockReturnValue(() => {}) onExit = vi.fn().mockReturnValue(() => {}) attach = vi.fn().mockResolvedValue(undefined) - attachForReconnect = vi.fn().mockResolvedValue({}) + attachForReconnect = attachForReconnectMock + setPtyDeliveryPauseAdapter = pauseAdapterMock dispose = vi.fn() } })) @@ -53,6 +103,7 @@ vi.mock('../ipc/pty', () => ({ clearPtyOwnershipForConnection: vi.fn(), clearProviderPtyState: vi.fn(), deletePtyOwnership: vi.fn(), + restorePtyIncarnation: vi.fn(), setPtyOwnership: vi.fn() })) @@ -67,15 +118,251 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ unregisterSshGitProvider: vi.fn() })) -const { registerSshPtyProvider } = await import('../ipc/pty') +const { + clearProviderPtyState, + clearPtyOwnershipForConnection, + deletePtyOwnership, + getSshPtyProvider, + getPtyIdsForConnection, + registerSshPtyProvider, + setPtyOwnership +} = await import('../ipc/pty') +const { closeSshPtyOutputGeneration } = await import('../ipc/ssh-pty-output-intake-registry') +const { applySshPtySourceCancellationProof } = await import('../ipc/ssh-pty-output-intake-registry') +const { applySshPtySourceRecoveryCancellationProof } = + await import('../ipc/ssh-pty-output-intake-registry') +const { getSshPtyAcceptedSourceCheckpoints } = await import('../ipc/ssh-pty-output-intake-registry') +const { installSshPtySourceAckPublisher } = await import('../ipc/ssh-pty-output-intake-registry') +const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') describe('SshRelaySession data delivery', () => { beforeEach(() => { vi.clearAllMocks() + ptyDataHandlerRef.current = undefined + attachForReconnectMock.mockResolvedValue({}) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([]) + vi.mocked(applySshPtySourceCancellationProof).mockReturnValue(true) + vi.mocked(applySshPtySourceRecoveryCancellationProof).mockReturnValue(true) + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease', + ...(options.outputFlowControl + ? { + outputFlowControl: { version: 1, windowSu: options.outputFlowControl.requestedWindowSu } + } + : {}) + })) muxRequestMock.mockResolvedValue([]) mockDeploySuccess() }) + async function runRecoverySequence(args: { + targetId: string + recoveryEndSu: number + recoveryFrame?: readonly [startSu: number, endSu: number] + liveFrame?: readonly [startSu: number, endSu: number] + }) { + muxRequestMock.mockImplementation(async (method) => + method === 'pty.cancelDelivery' + ? { canceled: true, sentEndSu: args.recoveryEndSu, creditedEndSu: 4 } + : [] + ) + let generation = 0 + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: ++generation, + ownerGeneration: generation, + ownerLease: `owner-lease-${generation}`, + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: `ssh:${args.targetId}@@pty-1`, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const deps = createMockDeps() + const session = new SshRelaySession( + args.targetId, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + await session.establish(deps.mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue([`ssh:${args.targetId}@@pty-1`]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + attachForReconnectMock.mockImplementation(async () => { + queueMicrotask(() => { + if (args.recoveryFrame) { + const [sourceStartSu, sourceEndSu] = args.recoveryFrame + ptyDataHandlerRef.current?.({ + id: `ssh:${args.targetId}@@pty-1`, + data: 'recovery', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: sourceEndSu - sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `new-token:${sourceStartSu}:${sourceEndSu}`, + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu, + sourceEndSu + } + }) + } + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: args.recoveryEndSu + }) + if (args.liveFrame) { + const [sourceStartSu, sourceEndSu] = args.liveFrame + ptyDataHandlerRef.current?.({ + id: `ssh:${args.targetId}@@pty-1`, + data: 'live', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: sourceEndSu - sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `new-token:${sourceStartSu}:${sourceEndSu}`, + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu, + sourceEndSu + } + }) + } + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: args.recoveryEndSu + } + } + }) + await session.reconnect(deps.mockConn) + return { ...deps, session } + } + + it('transfers negotiated owner recovery exactly once across an explicit detach', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, + platform: 'linux-x64', + serverBuildId: 'test-relay-build' + }) + const first = new SshRelaySession('recovery-target', getMainWindow, mockStore, mockPortForward) + + await first.establish(mockConn) + first.detach() + const second = new SshRelaySession('recovery-target', getMainWindow, mockStore, mockPortForward) + await second.establish(mockConn) + + const firstOpen = openConsumerSessionMock.mock.calls[0]?.[1] + const recoveredOpen = openConsumerSessionMock.mock.calls[1]?.[1] + expect(recoveredOpen).toMatchObject({ + clientInstanceId: firstOpen.clientInstanceId, + resume: { ownerGeneration: 1, ownerLease: 'test-owner-lease' } + }) + + second.dispose() + const fresh = new SshRelaySession('recovery-target', getMainWindow, mockStore, mockPortForward) + await fresh.establish(mockConn) + const freshOpen = openConsumerSessionMock.mock.calls[2]?.[1] + expect(freshOpen.clientInstanceId).not.toBe(firstOpen.clientInstanceId) + expect(freshOpen).not.toHaveProperty('resume') + fresh.dispose() + }) + + it('clears stale owner recovery and retries a fresh same-build relay once without resume', async () => { + const targetId = 'fresh-relay-retry' + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() }, + platform: 'linux-x64', + serverBuildId: 'test-relay-build' + }) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: `ssh:${targetId}@@pty-1`, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const first = new SshRelaySession(targetId, getMainWindow, mockStore, mockPortForward) + await first.establish(mockConn) + first.detach() + + openConsumerSessionMock + .mockRejectedValueOnce(createMismatchedOwnerRecoveryError()) + .mockImplementationOnce(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'fresh-owner-lease', + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + vi.mocked(getPtyIdsForConnection).mockReturnValue([`ssh:${targetId}@@pty-1`]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + attachForReconnectMock.mockResolvedValue({ + incarnationId: 'incarnation-1', + sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' } + }) + const second = new SshRelaySession(targetId, getMainWindow, mockStore, mockPortForward) + const openCallCountBeforeRetry = openConsumerSessionMock.mock.calls.length + + await second.establish(mockConn) + + const retryCalls = openConsumerSessionMock.mock.calls + .slice(openCallCountBeforeRetry) + .map(([, options]) => options) + expect(retryCalls).toHaveLength(2) + expect(retryCalls[0]).toHaveProperty('resume') + expect(retryCalls[1]).not.toHaveProperty('resume') + expect(attachForReconnectMock).toHaveBeenCalledWith( + 'pty-1', + undefined, + Object.freeze({ status: 'checkpointUnavailable' }) + ) + second.dispose() + }) + it('delivers empty transformed relay spans with raw sequence metadata', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const runtime = { onPtyData: vi.fn(() => 17), onPtyExit: vi.fn() } @@ -95,18 +382,425 @@ describe('SshRelaySession data delivery', () => { data: string sequenceChars?: number transformed?: boolean + providerGeneration: number + ptyIncarnation: string + source: { + relayPtyId: string + spanId: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string + sourceStartSu: number + sourceEndSu: number + } }) => void + const source = { + relayPtyId: 'pty-1', + spanId: 'token-1:0:9', + clientGeneration: 1, + ownerGeneration: 1, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 9 + } - onData({ id: 'ssh-pty-1', data: '', sequenceChars: 9, transformed: true }) - - expect(runtime.onPtyData).toHaveBeenCalledWith('ssh-pty-1', '', expect.any(Number), 9, true) - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:data', { + onData({ id: 'ssh-pty-1', data: '', sequenceChars: 9, transformed: true, - seq: 17, - rawLength: 9 + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + source }) + + expect(acceptOutputDataMock).toHaveBeenCalledWith({ + id: 'ssh-pty-1', + data: '', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + rawLength: 9, + transformed: true, + source + }) + expect(runtime.onPtyData).not.toHaveBeenCalled() + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:data', expect.anything()) + }) + + it('forwards negotiated source identity to the bounded intake exactly once', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + const ptyProvider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { + onData: ReturnType + } + const onData = ptyProvider.onData.mock.calls[0]?.[0] as (payload: { + id: string + data: string + providerGeneration: number + ptyIncarnation: string + source: { + relayPtyId: string + spanId: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string + sourceStartSu: number + sourceEndSu: number + } + }) => void + const source = { + relayPtyId: 'pty-1', + spanId: 'token-1:0:4', + clientGeneration: 1, + ownerGeneration: 1, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + + onData({ + id: 'ssh-pty-1', + data: 'data', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + source + }) + + expect(acceptOutputDataMock).toHaveBeenCalledOnce() + expect(acceptOutputDataMock).toHaveBeenCalledWith( + expect.objectContaining({ id: 'ssh-pty-1', rawLength: 4, source }) + ) + }) + + it('rejects missing negotiated source identity before main admission', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + const provider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { + onData: ReturnType + } + const onData = provider.onData.mock.calls[0]?.[0] as (payload: Record) => void + + onData({ + id: 'ssh-pty-1', + data: 'data', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1' + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(closeSshPtyOutputGeneration).toHaveBeenCalledWith( + 23, + 'ssh_source_frame_malformed_or_missing' + ) + }) + + it('keeps unoffered source metadata out of legacy intake', async () => { + openConsumerSessionMock.mockImplementationOnce(async (_mux, options) => ({ + mode: 'legacy-fallback', + clientInstanceId: options.clientInstanceId, + serverBuildId: 'test-relay-build' + })) + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + const provider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { + onData: ReturnType + } + const onData = provider.onData.mock.calls[0]?.[0] as (payload: Record) => void + + onData({ + id: 'ssh-pty-1', + data: 'data', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + source: { + relayPtyId: 'pty-1', + spanId: 'token-1:0:4', + clientGeneration: 1, + ownerGeneration: 1, + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + + expect(acceptOutputDataMock).toHaveBeenCalledWith( + expect.not.objectContaining({ source: expect.anything() }) + ) + }) + + it('keeps same-build method-not-found fallback token-free', async () => { + openConsumerSessionMock.mockImplementationOnce(async (_mux, options) => ({ + mode: 'legacy-fallback', + clientInstanceId: options.clientInstanceId, + serverBuildId: 'test-relay-build' + })) + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + + await session.establish(mockConn) + + expect(session.getState()).toBe('ready') + expect(pauseAdapterMock).not.toHaveBeenCalled() + expect(openConsumerSessionMock.mock.calls[0][1]).toHaveProperty('outputFlowControl') + expect(installSshPtySourceAckPublisher).not.toHaveBeenCalled() + }) + + it('publishes negotiated ACK batches through mux settlement', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + const publish = vi.mocked(installSshPtySourceAckPublisher).mock.calls[0]?.[1] + const settled = vi.fn() + const batch = { + acknowledgements: [ + { + id: 'pty-1', + clientGeneration: 1, + ownerGeneration: 1, + deliveryToken: 'token-1', + creditedEndSu: 4 + } + ] + } + + publish?.(batch, settled) + + expect(openConsumerSessionMock.mock.calls[0][1]).toMatchObject({ + outputFlowControl: { requestedWindowSu: 256 * 1024 } + }) + expect(deployAndLaunchRelay).toHaveBeenCalledWith(mockConn, undefined, undefined, 'target-1') + expect(notifyWithSettlementMock).toHaveBeenCalledWith('pty.ackData', batch, settled) + }) + + it('offers V1 through reconnect negotiation', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + + await session.reconnect(mockConn) + + expect(openConsumerSessionMock.mock.calls[0][1]).toHaveProperty('outputFlowControl') + expect(openConsumerSessionMock.mock.calls[1][1]).toHaveProperty('outputFlowControl') + expect(deployAndLaunchRelay).toHaveBeenNthCalledWith( + 1, + mockConn, + undefined, + undefined, + 'target-1' + ) + expect(deployAndLaunchRelay).toHaveBeenNthCalledWith( + 2, + mockConn, + undefined, + undefined, + 'target-1' + ) + }) + + it('reattaches V1 from an exact checkpoint and quarantines live data until recoveryComplete', async () => { + let generation = 0 + openConsumerSessionMock.mockImplementation(async (_mux, options) => { + generation++ + return { + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: generation, + ownerGeneration: generation, + ownerLease: `owner-lease-${generation}`, + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + } + }) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: 'ssh:target-1@@pty-1', + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue(['ssh:target-1@@pty-1']) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + let transferDisposedMux = false + const publishHeldRecovery = (sink: (payload: unknown) => void): void => { + for (const [data, sourceStartSu, sourceEndSu] of [ + ['re', 4, 6], + ['co', 6, 8], + ['live', 8, 12] + ] as const) { + sink({ + id: 'ssh:target-1@@pty-1', + data, + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: sourceEndSu - sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `new-token:${sourceStartSu}:${sourceEndSu}`, + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu, + sourceEndSu + } + }) + } + } + const recoveryActivationLease = { + commit: vi.fn(), + retire: vi.fn() + } + const sourceActivationLease = { + commit: vi.fn(), + rollback: vi.fn(async () => true), + transferToRecovery: vi.fn((sink: (payload: unknown) => void) => { + publishHeldRecovery(sink) + return recoveryActivationLease + }) + } + attachForReconnectMock.mockImplementation(async () => { + const canceled = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.deliveryCanceled' + )?.[1] as ((params: Record) => void) | undefined + const disposeCount = muxDisposeMock.mock.calls.length + canceled?.({ + id: 'pty-1', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + replacementDeliveryToken: 'new-token' + }) + transferDisposedMux = muxDisposeMock.mock.calls.length !== disposeCount + queueMicrotask(() => { + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }) + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }, + sourceActivationLease + } + }) + + await session.reconnect(mockConn) + + expect(attachForReconnectMock).toHaveBeenCalledWith( + 'pty-1', + undefined, + expect.objectContaining({ + status: 'checkpoint', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + }) + ) + expect(acceptOutputDataMock.mock.calls.map(([payload]) => payload.data)).toEqual([ + 're', + 'co', + 'live' + ]) + expect(transferDisposedMux).toBe(false) + expect(sourceActivationLease.transferToRecovery).toHaveBeenCalledOnce() + expect(sourceActivationLease.commit).not.toHaveBeenCalled() + expect(sourceActivationLease.rollback).not.toHaveBeenCalled() + expect(recoveryActivationLease.commit).toHaveBeenCalledOnce() + expect(recoveryActivationLease.retire).not.toHaveBeenCalled() + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:replay', expect.anything()) + + const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length + const canceled = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.deliveryCanceled' + )?.[1] as ((params: Record) => void) | undefined + canceled?.({ + id: 'pty-1', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token' + }) + expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount) + }) + + it.each([ + ['gap', [5, 8] as const], + ['overlap', [3, 8] as const], + ['incomplete suffix', [4, 6] as const], + ['missing body', undefined] + ])('rejects %s recovery without destroying the physical PTY or lease', async (label, frame) => { + const targetId = `invalid-recovery-${label.replace(' ', '-')}` + const { mockStore, mockWindow } = await runRecoverySequence({ + targetId, + recoveryEndSu: 8, + ...(frame ? { recoveryFrame: frame } : {}) + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(muxRequestMock).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token' + }) + expect(applySshPtySourceRecoveryCancellationProof).toHaveBeenCalledWith( + { + id: `ssh:${targetId}@@pty-1`, + code: -1, + providerGeneration: 23, + ptyIncarnation: 'incarnation-1' + }, + { sentEndSu: 8, creditedEndSu: 4 } + ) + expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'detached') + expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith(targetId, 'pty-1', 'expired') + expect(mockStore.markSshRemotePtyLeases).not.toHaveBeenCalled() + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(setPtyOwnership).not.toHaveBeenCalled() + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + }) + + it('accepts empty recovery only when the checkpoint equals the recovery end', async () => { + const { mockStore } = await runRecoverySequence({ + targetId: 'empty-recovery', + recoveryEndSu: 4, + liveFrame: [4, 8] + }) + + expect(acceptOutputDataMock.mock.calls.map(([payload]) => payload.data)).toEqual(['live']) + expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( + 'empty-recovery', + 'pty-1', + 'attached' + ) }) }) diff --git a/src/main/ssh/ssh-relay-session-incarnation.test.ts b/src/main/ssh/ssh-relay-session-incarnation.test.ts index a98180308b2..129a98b3ebc 100644 --- a/src/main/ssh/ssh-relay-session-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-incarnation.test.ts @@ -2,15 +2,43 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { SshRelaySession } from './ssh-relay-session' import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' -const { muxRequestMock } = vi.hoisted(() => ({ muxRequestMock: vi.fn() })) +const { acceptOutputExitMock, muxRequestMock } = vi.hoisted(() => ({ + acceptOutputExitMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn() +})) vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: vi.fn(async (_mux, options) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + })) +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: vi.fn().mockResolvedValue(undefined), + acceptSshPtyOutputExit: acceptOutputExitMock, + allocateSshPtyProviderGeneration: vi.fn(() => 31), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + applySshPtySourceCancellationProof: vi.fn(() => true), + applySshPtySourceRecoveryCancellationProof: vi.fn(() => true), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}) +})) vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) vi.mock('./ssh-channel-multiplexer', () => ({ SshChannelMultiplexer: class MockSshChannelMultiplexer { notify = vi.fn() + notifyWithSettlement = vi.fn() request = muxRequestMock onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) onRequest = vi.fn().mockReturnValue(() => {}) onDispose = vi.fn().mockReturnValue(() => {}) dispose = vi.fn() @@ -90,22 +118,41 @@ describe('SSH relay PTY incarnation exits', () => { id: string code: number incarnationId: string + providerGeneration: number + ptyIncarnation: string }) => void vi.mocked(isCurrentPtyExit).mockReturnValueOnce(false) - onExit({ id: 'ssh:target-1@@pty-reused', code: 0, incarnationId: 'old-incarnation' }) + onExit({ + id: 'ssh:target-1@@pty-reused', + code: 0, + incarnationId: 'old-incarnation', + providerGeneration: 31, + ptyIncarnation: 'old-incarnation' + }) expect(clearProviderPtyState).not.toHaveBeenCalled() expect(deletePtyOwnership).not.toHaveBeenCalled() expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalled() + expect(acceptOutputExitMock).not.toHaveBeenCalled() expect(runtime.onPtyExit).not.toHaveBeenCalled() expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) - onExit({ id: 'ssh:target-1@@pty-reused', code: 7, incarnationId: 'current-incarnation' }) - expect(runtime.onPtyExit).toHaveBeenCalledWith( - 'ssh:target-1@@pty-reused', - 7, - 'current-incarnation' + onExit({ + id: 'ssh:target-1@@pty-reused', + code: 7, + incarnationId: 'current-incarnation', + providerGeneration: 31, + ptyIncarnation: 'current-incarnation' + }) + await vi.waitFor(() => + expect(acceptOutputExitMock).toHaveBeenCalledWith({ + id: 'ssh:target-1@@pty-reused', + code: 7, + providerGeneration: 31, + ptyIncarnation: 'current-incarnation' + }) ) + expect(runtime.onPtyExit).not.toHaveBeenCalled() }) }) diff --git a/src/main/ssh/ssh-relay-session-model-migration.test.ts b/src/main/ssh/ssh-relay-session-model-migration.test.ts new file mode 100644 index 00000000000..9bd2216357d --- /dev/null +++ b/src/main/ssh/ssh-relay-session-model-migration.test.ts @@ -0,0 +1,263 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { SshRelaySession } from './ssh-relay-session' +import { + createMismatchedOwnerRecoveryError, + createMockDeps, + mockDeploySuccess +} from './ssh-relay-session-test-fixtures' + +type SettledMigration = { + status: 'settled' + checkpoint: { + id: string + providerGeneration: number + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string + acceptedSourceEndSu: number + } +} + +const { attachForReconnectMock, beginMigrationMock, muxRequestMock, openConsumerSessionMock } = + vi.hoisted(() => ({ + attachForReconnectMock: vi.fn().mockResolvedValue({}), + beginMigrationMock: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + muxRequestMock: vi.fn(), + openConsumerSessionMock: vi.fn() + })) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: openConsumerSessionMock +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: vi.fn().mockResolvedValue(undefined), + acceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined), + allocateSshPtyProviderGeneration: vi.fn(() => 23), + beginSshPtyOutputGenerationMigration: beginMigrationMock, + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}), + applySshPtySourceCancellationProof: vi.fn(), + applySshPtySourceRecoveryCancellationProof: vi.fn() +})) + +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = muxRequestMock + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = vi.fn() + isDisposed = vi.fn().mockReturnValue(false) + } +})) + +vi.mock('../providers/ssh-pty-provider', () => ({ + SshPtyProvider: class MockSshPtyProvider { + onData = vi.fn().mockReturnValue(() => {}) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockReturnValue(() => {}) + attach = vi.fn().mockResolvedValue(undefined) + attachForReconnect = attachForReconnectMock + setPtyDeliveryPauseAdapter = vi.fn() + dispose = vi.fn() + } +})) + +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../ipc/pty', () => ({ + registerSshPtyProvider: vi.fn(), + unregisterSshPtyProvider: vi.fn(), + getSshPtyProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }), + getPtyIdsForConnection: vi.fn().mockReturnValue([]), + clearPtyOwnershipForConnection: vi.fn(), + clearProviderPtyState: vi.fn(), + deletePtyOwnership: vi.fn(), + restorePtyIncarnation: vi.fn(), + setPtyOwnership: vi.fn() +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { getPtyIdsForConnection, getSshPtyProvider, registerSshPtyProvider } = + await import('../ipc/pty') +const { getSshPtyAcceptedSourceCheckpoints } = await import('../ipc/ssh-pty-output-intake-registry') + +function pendingMigration() { + let resolve!: (result: SettledMigration) => void + const result = new Promise((promiseResolve) => { + resolve = promiseResolve + }) + return { result, resolve } +} + +function settledMigration(appPtyId: string, acceptedSourceEndSu: number): SettledMigration { + return { + status: 'settled', + checkpoint: { + id: appPtyId, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu + } + } +} + +describe('SshRelaySession model migration', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([]) + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'owner-lease-1', + ...(options.outputFlowControl + ? { + outputFlowControl: { version: 1, windowSu: options.outputFlowControl.requestedWindowSu } + } + : {}) + })) + muxRequestMock.mockResolvedValue([]) + mockDeploySuccess() + }) + + it('keeps overlapping reconnects behind the old-generation model migration', async () => { + const targetId = 'migration-fence' + const appPtyId = `ssh:${targetId}@@pty-1` + const migration = pendingMigration() + beginMigrationMock.mockReturnValueOnce({ + byPty: new Map([[appPtyId, migration.result]]), + completion: migration.result.then(() => {}) + }) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: appPtyId, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const deps = createMockDeps() + const session = new SshRelaySession( + targetId, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + await session.establish(deps.mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue([appPtyId]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + attachForReconnectMock.mockResolvedValue({ + incarnationId: 'incarnation-1', + sourceRecovery: { status: 'restoreRequired', reason: 'relayRestoreRequired' } + }) + + const firstReconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => expect(registerSshPtyProvider).toHaveBeenCalledTimes(2)) + const secondReconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => expect(beginMigrationMock).toHaveBeenCalledTimes(2)) + expect(attachForReconnectMock).not.toHaveBeenCalled() + + migration.resolve(settledMigration(appPtyId, 8)) + await Promise.all([firstReconnect, secondReconnect]) + + expect(attachForReconnectMock).toHaveBeenCalledWith( + 'pty-1', + undefined, + Object.freeze({ + status: 'checkpoint', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 8 + }) + ) + }) + + it('waits for a stale-owner migration before requesting restore', async () => { + const targetId = 'migration-stale-owner' + const appPtyId = `ssh:${targetId}@@pty-1` + const migration = pendingMigration() + beginMigrationMock.mockReturnValueOnce({ + byPty: new Map([[appPtyId, migration.result]]), + completion: migration.result.then(() => {}) + }) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + settledMigration(appPtyId, 4).checkpoint + ]) + const deps = createMockDeps() + const session = new SshRelaySession( + targetId, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + await session.establish(deps.mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue([appPtyId]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + openConsumerSessionMock + .mockRejectedValueOnce(createMismatchedOwnerRecoveryError()) + .mockImplementationOnce(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: 2, + ownerGeneration: 2, + ownerLease: 'fresh-owner-lease', + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + attachForReconnectMock.mockResolvedValue({ + incarnationId: 'incarnation-1', + sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' } + }) + + const reconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => expect(openConsumerSessionMock).toHaveBeenCalledTimes(3)) + expect(attachForReconnectMock).not.toHaveBeenCalled() + + migration.resolve(settledMigration(appPtyId, 8)) + await reconnect + + expect(attachForReconnectMock).toHaveBeenCalledWith( + 'pty-1', + undefined, + Object.freeze({ status: 'checkpointUnavailable' }) + ) + }) +}) diff --git a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts index cd3deff182b..c624e72564d 100644 --- a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts @@ -8,13 +8,43 @@ type MockMuxInstance = { requestHandlers: Map) => Promise> } -const { muxRequestMock, muxInstancesRaw } = vi.hoisted(() => ({ - muxRequestMock: vi.fn(), - muxInstancesRaw: [] as unknown[] -})) +const { acceptOutputExitMock, muxRequestMock, openConsumerSessionMock, muxInstancesRaw } = + vi.hoisted(() => ({ + acceptOutputExitMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn(), + openConsumerSessionMock: vi.fn( + async ( + _mux: unknown, + options: { clientInstanceId: string; outputFlowControl?: unknown } + ) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + }) + ), + muxInstancesRaw: [] as unknown[] + })) const muxInstances = muxInstancesRaw as MockMuxInstance[] vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: openConsumerSessionMock +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: vi.fn().mockResolvedValue(undefined), + acceptSshPtyOutputExit: acceptOutputExitMock, + allocateSshPtyProviderGeneration: vi.fn(() => 17), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + applySshPtySourceRecoveryCancellationProof: vi.fn(() => true), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}) +})) vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) vi.mock('node:crypto', async (importOriginal) => { const actual = await importOriginal() @@ -27,8 +57,10 @@ vi.mock('./ssh-channel-multiplexer', () => ({ SshChannelMultiplexer: class MockSshChannelMultiplexer { requestHandlers = new Map) => Promise>() notify = vi.fn() + notifyWithSettlement = vi.fn() request = muxRequestMock onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) onRequest = vi.fn( (method: string, handler: (params: Record) => Promise) => { this.requestHandlers.set(method, handler) @@ -97,6 +129,7 @@ const { setPtyOwnership, restorePtyIncarnation } = await import('../ipc/pty') +const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') const APP_PTY_ID = 'ssh:target-1@@pty-live' const INCARNATION_LEAF_ID = '11111111-1111-4111-8111-111111111111' @@ -112,14 +145,26 @@ function detachedLease() { } } -function emitExitDuringAttach(payload: { id: string; code: number; incarnationId?: string }): void { +function emitExitDuringAttach(payload: { + id: string + code: number + incarnationId?: string + providerGeneration?: number + ptyIncarnation?: string +}): void { const registeredProvider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { onExit: ReturnType } const exitHandler = registeredProvider.onExit.mock.calls[0]?.[0] as | ((exit: typeof payload) => void) | undefined - queueMicrotask(() => exitHandler?.(payload)) + queueMicrotask(() => + exitHandler?.({ + providerGeneration: 17, + ptyIncarnation: payload.incarnationId ?? `legacy:${payload.id}`, + ...payload + }) + ) } describe('SshRelaySession reconnect incarnation ordering', () => { @@ -135,7 +180,152 @@ describe('SshRelaySession reconnect incarnation ordering', () => { vi.mocked(getPtyIdsForConnection).mockReturnValue([]) }) + it('bounds fifty reattaches to eight workers without slow or failed sibling head-of-line delay', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + vi.clearAllMocks() + mockDeploySuccess() + + const ptyIds = Array.from({ length: 50 }, (_, index) => `pty-${index}`) + let active = 0 + let peakActive = 0 + const attempts = new Map() + const mockAttach = vi.fn((id: string) => { + attempts.set(id, (attempts.get(id) ?? 0) + 1) + if (id === 'pty-0') { + return new Promise(() => {}) + } + if (id === 'pty-1') { + return Promise.reject(new Error('isolated attach failure')) + } + active++ + peakActive = Math.max(peakActive, active) + return new Promise((resolve) => { + setTimeout(() => { + active-- + resolve() + }, 100) + }) + }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue(ptyIds) + const random = vi.spyOn(Math, 'random').mockReturnValue(0) + vi.useFakeTimers() + + try { + const reconnect = session.reconnect(mockConn) + await vi.advanceTimersByTimeAsync(750) + + expect( + vi + .mocked(mockStore.markSshRemotePtyLease) + .mock.calls.filter(([, , state]) => state === 'attached') + .map(([, id]) => id) + ).toHaveLength(48) + expect(peakActive).toBeLessThanOrEqual(8) + + await vi.advanceTimersByTimeAsync(20_000) + await reconnect + + expect(mockAttach).toHaveBeenCalledTimes(52) + expect(attempts.get('pty-0')).toBe(2) + expect(attempts.get('pty-1')).toBe(2) + expect(session.getState()).toBe('ready') + expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + 'target-1', + expect.any(String), + 'expired' + ) + } finally { + vi.useRealTimers() + random.mockRestore() + } + }) + + it('offers source credit through reconnect', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + + await session.establish(mockConn) + await session.reconnect(mockConn) + + expect(deployAndLaunchRelay).toHaveBeenNthCalledWith( + 1, + mockConn, + undefined, + undefined, + 'target-1' + ) + expect(deployAndLaunchRelay).toHaveBeenNthCalledWith( + 2, + mockConn, + undefined, + undefined, + 'target-1' + ) + expect(openConsumerSessionMock).toHaveBeenCalledTimes(2) + for (const [, options] of openConsumerSessionMock.mock.calls) { + expect(options.outputFlowControl).toBeDefined() + } + }) + + it('rolls back a timed-out activation that resolves after its replacement commits', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + vi.clearAllMocks() + mockDeploySuccess() + const timedOutLease = { commit: vi.fn(), rollback: vi.fn() } + const replacementLease = { commit: vi.fn(), rollback: vi.fn() } + let resolveTimedOut!: (result: { + incarnationId: string + sourceActivationLease: typeof timedOutLease + }) => void + const mockAttach = vi + .fn() + .mockReturnValueOnce( + new Promise((resolve) => { + resolveTimedOut = resolve + }) + ) + .mockResolvedValueOnce({ + incarnationId: 'incarnation-replacement', + sourceActivationLease: replacementLease + }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1']) + const random = vi.spyOn(Math, 'random').mockReturnValue(0) + vi.useFakeTimers() + + try { + const reconnect = session.reconnect(mockConn) + await vi.advanceTimersByTimeAsync(11_000) + await reconnect + resolveTimedOut({ + incarnationId: 'incarnation-timed-out', + sourceActivationLease: timedOutLease + }) + await Promise.resolve() + + expect(replacementLease.commit).toHaveBeenCalledOnce() + expect(replacementLease.rollback).not.toHaveBeenCalled() + expect(timedOutLease.rollback).toHaveBeenCalledOnce() + expect(timedOutLease.commit).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + random.mockRestore() + } + }) + it('keeps the winning reconnect incarnation when a stale health check resolves last', async () => { + const consumerInstanceId = '00000000-0000-4000-8000-000000000000' const initialIncarnation = '00000000-0000-4000-8000-000000000001' const winningIncarnation = '00000000-0000-4000-8000-000000000002' const staleIncarnation = '00000000-0000-4000-8000-000000000003' @@ -152,6 +342,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => { return resolveHomeCalls === 2 ? staleHealthCheck : Promise.resolve('/') }) vi.mocked(randomUUID) + .mockReturnValueOnce(consumerInstanceId) .mockReturnValueOnce(initialIncarnation) .mockReturnValueOnce(winningIncarnation) .mockReturnValue(staleIncarnation) @@ -190,7 +381,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => { 'target-1', winningIncarnation ) - expect(randomUUID).toHaveBeenCalledTimes(2) + expect(randomUUID).toHaveBeenCalledTimes(3) }) it('restores and persists exact incarnation proof from reconnect attach', async () => { @@ -242,11 +433,12 @@ describe('SshRelaySession reconnect incarnation ordering', () => { onPtySpawned: vi.fn(), registerPty: vi.fn() } + const sourceActivationLease = { commit: vi.fn(), rollback: vi.fn() } vi.mocked(getSshPtyProvider).mockReturnValue({ attachForReconnect: vi.fn().mockImplementation(async () => { emitExitDuringAttach({ id: APP_PTY_ID, code: 0, incarnationId }) emitExitDuringAttach({ id: APP_PTY_ID, code: 0, incarnationId: 'incarnation-stale' }) - return { incarnationId, replay: 'dead-output' } + return { incarnationId, replay: 'dead-output', sourceActivationLease } }), dispose: vi.fn() } as unknown as ReturnType) @@ -263,12 +455,20 @@ describe('SshRelaySession reconnect incarnation ordering', () => { await session.establish(mockConn) - expect(runtime.onPtyExit).toHaveBeenCalledWith(APP_PTY_ID, 0, incarnationId) + expect(acceptOutputExitMock).toHaveBeenCalledWith({ + id: APP_PTY_ID, + code: 0, + providerGeneration: 17, + ptyIncarnation: incarnationId + }) + expect(runtime.onPtyExit).not.toHaveBeenCalled() expect(runtime.acceptPtyIncarnationForExit).toHaveBeenCalledWith(APP_PTY_ID, incarnationId) expect(runtime.registerPty).not.toHaveBeenCalled() expect(restorePtyIncarnation).toHaveBeenCalledWith(APP_PTY_ID, incarnationId) expect(setPtyOwnership).not.toHaveBeenCalled() expect(mockStore.persistPtyBinding).not.toHaveBeenCalled() + expect(sourceActivationLease.rollback).toHaveBeenCalledOnce() + expect(sourceActivationLease.commit).not.toHaveBeenCalled() expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith( 'target-1', 'pty-live', @@ -290,6 +490,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => { onPtySpawned: vi.fn(), registerPty: vi.fn() } + const sourceActivationLease = { commit: vi.fn(), rollback: vi.fn() } vi.mocked(getSshPtyProvider).mockReturnValue({ attachForReconnect: vi.fn().mockImplementation(async () => { emitExitDuringAttach({ @@ -297,7 +498,7 @@ describe('SshRelaySession reconnect incarnation ordering', () => { code: 0, incarnationId: 'incarnation-old' }) - return { incarnationId: currentIncarnationId, replay: 'live-output' } + return { incarnationId: currentIncarnationId, replay: 'live-output', sourceActivationLease } }), dispose: vi.fn() } as unknown as ReturnType) @@ -316,6 +517,8 @@ describe('SshRelaySession reconnect incarnation ordering', () => { expect(runtime.onPtyExit).not.toHaveBeenCalled() expect(runtime.acceptPtyIncarnationForExit).not.toHaveBeenCalled() + expect(sourceActivationLease.commit).toHaveBeenCalledOnce() + expect(sourceActivationLease.rollback).not.toHaveBeenCalled() expect(runtime.registerPty).toHaveBeenCalledWith(APP_PTY_ID, 'worktree-1', 'target-1', { tabId: 'tab-1', leafId: INCARNATION_LEAF_ID, diff --git a/src/main/ssh/ssh-relay-session-recovery-races.test.ts b/src/main/ssh/ssh-relay-session-recovery-races.test.ts new file mode 100644 index 00000000000..711f926e056 --- /dev/null +++ b/src/main/ssh/ssh-relay-session-recovery-races.test.ts @@ -0,0 +1,791 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { SshRelaySession } from './ssh-relay-session' +import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' + +const { + acceptOutputDataMock, + acceptOutputExitMock, + muxRequestMock, + onNotificationByMethodMock, + openConsumerSessionMock, + muxDisposeMock, + ptyProviderDisposeMock, + sourceAckCleanupMock, + sourceCancellationCleanupMock, + attachForReconnectMock, + ptyDataHandlerRef, + ptyExitHandlerRef +} = vi.hoisted(() => ({ + acceptOutputDataMock: vi.fn().mockResolvedValue(undefined), + acceptOutputExitMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn(), + onNotificationByMethodMock: vi.fn(), + openConsumerSessionMock: vi.fn(), + muxDisposeMock: vi.fn(), + ptyProviderDisposeMock: vi.fn(), + sourceAckCleanupMock: vi.fn(), + sourceCancellationCleanupMock: vi.fn(), + attachForReconnectMock: vi.fn().mockResolvedValue({}), + ptyDataHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) }, + ptyExitHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) } +})) + +vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: openConsumerSessionMock +})) +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: acceptOutputDataMock, + acceptSshPtyOutputExit: acceptOutputExitMock, + allocateSshPtyProviderGeneration: vi.fn(() => 23), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + installSshPtySourceAckPublisher: vi.fn(() => sourceAckCleanupMock), + installSshPtySourceCancellationPublisher: vi.fn(() => sourceCancellationCleanupMock), + applySshPtySourceCancellationProof: vi.fn(() => true), + applySshPtySourceRecoveryCancellationProof: vi.fn(() => true) +})) +vi.mock('./ssh-channel-multiplexer', () => ({ + SshChannelMultiplexer: class MockSshChannelMultiplexer { + notify = vi.fn() + notifyWithSettlement = vi.fn() + request = muxRequestMock + onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = onNotificationByMethodMock.mockImplementation(() => () => {}) + onRequest = vi.fn().mockReturnValue(() => {}) + onDispose = vi.fn().mockReturnValue(() => {}) + dispose = muxDisposeMock + isDisposed = vi.fn().mockReturnValue(false) + } +})) +vi.mock('../providers/ssh-pty-provider', () => ({ + isSshPtyNotFoundError: vi.fn().mockReturnValue(false), + isSshPtyIdentityMismatchError: vi.fn().mockReturnValue(false), + SshPtyProvider: class MockSshPtyProvider { + onData = vi.fn().mockImplementation((handler) => { + ptyDataHandlerRef.current = handler + return () => {} + }) + onReplay = vi.fn().mockReturnValue(() => {}) + onExit = vi.fn().mockImplementation((handler) => { + ptyExitHandlerRef.current = handler + return () => {} + }) + attachForReconnect = attachForReconnectMock + setPtyDeliveryPauseAdapter = vi.fn() + dispose = ptyProviderDisposeMock + } +})) +vi.mock('../providers/ssh-filesystem-provider', () => ({ + SshFilesystemProvider: class MockSshFilesystemProvider { + dispose = vi.fn() + } +})) +vi.mock('../providers/ssh-git-provider', () => ({ + SshGitProvider: class MockSshGitProvider {} +})) +vi.mock('../ipc/pty', () => ({ + registerSshPtyProvider: vi.fn(), + unregisterSshPtyProvider: vi.fn(), + getSshPtyProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }), + getPtyIdsForConnection: vi.fn().mockReturnValue([]), + isCurrentPtyExit: vi.fn(() => true), + clearPtyOwnershipForConnection: vi.fn(), + clearProviderPtyState: vi.fn(), + deletePtyOwnership: vi.fn(), + restorePtyIncarnation: vi.fn(), + setPtyOwnership: vi.fn() +})) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + registerSshFilesystemProvider: vi.fn(), + unregisterSshFilesystemProvider: vi.fn(), + getSshFilesystemProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }) +})) +vi.mock('../providers/ssh-git-dispatch', () => ({ + registerSshGitProvider: vi.fn(), + unregisterSshGitProvider: vi.fn() +})) + +const { + clearProviderPtyState, + clearPtyOwnershipForConnection, + deletePtyOwnership, + getSshPtyProvider, + getPtyIdsForConnection, + registerSshPtyProvider, + setPtyOwnership, + unregisterSshPtyProvider +} = await import('../ipc/pty') +const { closeSshPtyOutputGeneration, getSshPtyAcceptedSourceCheckpoints } = + await import('../ipc/ssh-pty-output-intake-registry') +const { applySshPtySourceCancellationProof } = await import('../ipc/ssh-pty-output-intake-registry') +const { applySshPtySourceRecoveryCancellationProof } = + await import('../ipc/ssh-pty-output-intake-registry') + +describe('SshRelaySession recovery race fencing', () => { + beforeEach(() => { + vi.clearAllMocks() + ptyDataHandlerRef.current = undefined + ptyExitHandlerRef.current = undefined + attachForReconnectMock.mockResolvedValue({}) + vi.mocked(getPtyIdsForConnection).mockReturnValue([]) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([]) + vi.mocked(applySshPtySourceCancellationProof).mockReturnValue(true) + vi.mocked(applySshPtySourceRecoveryCancellationProof).mockReturnValue(true) + muxRequestMock.mockResolvedValue([]) + mockDeploySuccess() + }) + + function emitSourceFrame(args: { + targetId: string + token: string + clientGeneration: number + ownerGeneration: number + sourceStartSu: number + sourceEndSu: number + }): void { + ptyDataHandlerRef.current?.({ + id: `ssh:${args.targetId}@@pty-1`, + data: 'late', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: args.sourceEndSu - args.sourceStartSu, + source: { + relayPtyId: 'pty-1', + spanId: `${args.token}:${args.sourceStartSu}:${args.sourceEndSu}`, + clientGeneration: args.clientGeneration, + ownerGeneration: args.ownerGeneration, + deliveryToken: args.token, + sourceStartSu: args.sourceStartSu, + sourceEndSu: args.sourceEndSu + } + }) + } + + async function prepareRecovery(targetId: string): Promise<{ + session: SshRelaySession + deps: ReturnType + }> { + let generation = 0 + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'negotiated', + clientInstanceId: options.clientInstanceId, + clientGeneration: ++generation, + ownerGeneration: generation, + ownerLease: `owner-lease-${generation}`, + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + })) + vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([ + { + id: `ssh:${targetId}@@pty-1`, + providerGeneration: 23, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + } + ]) + const deps = createMockDeps() + const session = new SshRelaySession( + targetId, + deps.getMainWindow, + deps.mockStore, + deps.mockPortForward + ) + await session.establish(deps.mockConn) + vi.mocked(getPtyIdsForConnection).mockReturnValue([`ssh:${targetId}@@pty-1`]) + vi.mocked(getSshPtyProvider).mockImplementation( + () => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1] + ) + return { session, deps } + } + + it('publishes held recovery data before an exact exit without waiting for completion', async () => { + const targetId = 'exit-with-complete-private-body' + const { session, deps } = await prepareRecovery(targetId) + const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() } + const sourceActivationLease = { + commit: vi.fn(), + rollback: vi.fn(async () => true), + transferToRecovery: vi.fn((sink: (payload: unknown) => void) => { + sink({ + id: `ssh:${targetId}@@pty-1`, + data: 'held', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: 4, + source: { + relayPtyId: 'pty-1', + spanId: 'new-token:4:8', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu: 4, + sourceEndSu: 8 + } + }) + return recoveryActivationLease + }) + } + attachForReconnectMock.mockImplementation(async () => { + ptyExitHandlerRef.current?.({ + id: `ssh:${targetId}@@pty-1`, + code: 0, + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + incarnationId: 'incarnation-1' + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }, + sourceActivationLease + } + }) + + await session.reconnect(deps.mockConn) + + expect(acceptOutputDataMock).toHaveBeenCalledWith(expect.objectContaining({ data: 'held' })) + expect(acceptOutputExitMock).toHaveBeenCalledOnce() + expect(acceptOutputDataMock.mock.invocationCallOrder[0]).toBeLessThan( + acceptOutputExitMock.mock.invocationCallOrder[0]! + ) + expect(sourceActivationLease.transferToRecovery).toHaveBeenCalledOnce() + expect(sourceActivationLease.rollback).not.toHaveBeenCalled() + expect(recoveryActivationLease.commit).toHaveBeenCalledOnce() + expect(recoveryActivationLease.retire).not.toHaveBeenCalled() + expect(muxRequestMock).not.toHaveBeenCalledWith('pty.cancelDelivery', expect.anything()) + expect(setPtyOwnership).not.toHaveBeenCalled() + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( + targetId, + 'pty-1', + 'attached' + ) + }) + + it('settles exact cancellation before publishing an exit with incomplete recovery data', async () => { + const targetId = 'exit-with-incomplete-private-body' + let settleCancellation: ((proof: unknown) => void) | undefined + muxRequestMock.mockImplementation((method) => + method === 'pty.cancelDelivery' + ? new Promise((resolve) => { + settleCancellation = resolve + }) + : Promise.resolve([]) + ) + const { session, deps } = await prepareRecovery(targetId) + const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() } + const sourceActivationLease = { + commit: vi.fn(), + rollback: vi.fn(async () => true), + transferToRecovery: vi.fn((sink: (payload: unknown) => void) => { + sink({ + id: `ssh:${targetId}@@pty-1`, + data: 'partial', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: 2, + source: { + relayPtyId: 'pty-1', + spanId: 'new-token:4:6', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu: 4, + sourceEndSu: 6 + } + }) + return recoveryActivationLease + }) + } + attachForReconnectMock.mockImplementation(async () => { + ptyExitHandlerRef.current?.({ + id: `ssh:${targetId}@@pty-1`, + code: 0, + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + incarnationId: 'incarnation-1' + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }, + sourceActivationLease + } + }) + + const reconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => + expect(muxRequestMock).toHaveBeenCalledWith('pty.cancelDelivery', { + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token' + }) + ) + expect(acceptOutputExitMock).not.toHaveBeenCalled() + + settleCancellation?.({ canceled: true, sentEndSu: 6, creditedEndSu: 4 }) + await reconnect + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(acceptOutputExitMock).toHaveBeenCalledOnce() + expect(recoveryActivationLease.commit).not.toHaveBeenCalled() + expect(recoveryActivationLease.retire).toHaveBeenCalledOnce() + expect(sourceActivationLease.rollback).not.toHaveBeenCalled() + expect(applySshPtySourceRecoveryCancellationProof).toHaveBeenCalledOnce() + }) + + it('awaits provisional cancellation proof before publishing an exact recovery exit', async () => { + const targetId = 'exit-before-recovery-identity' + let settleRollback: ((settled: boolean) => void) | undefined + const { session, deps } = await prepareRecovery(targetId) + const sourceActivationLease = { + commit: vi.fn(), + rollback: vi.fn( + () => + new Promise((resolve) => { + settleRollback = resolve + }) + ), + transferToRecovery: vi.fn() + } + attachForReconnectMock.mockImplementation(async () => { + ptyExitHandlerRef.current?.({ + id: `ssh:${targetId}@@pty-1`, + code: 0, + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + incarnationId: 'incarnation-1' + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' }, + sourceActivationLease + } + }) + + const reconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => expect(sourceActivationLease.rollback).toHaveBeenCalledOnce()) + expect(acceptOutputExitMock).not.toHaveBeenCalled() + + settleRollback?.(true) + await reconnect + + expect(acceptOutputExitMock).toHaveBeenCalledOnce() + expect(sourceActivationLease.transferToRecovery).not.toHaveBeenCalled() + expect(sourceActivationLease.commit).not.toHaveBeenCalled() + }) + + it('retains the empty recovery end as the first post-activation source anchor', async () => { + const targetId = 'empty-recovery-gap' + const { session, deps } = await prepareRecovery(targetId) + attachForReconnectMock.mockImplementation(async () => { + queueMicrotask(() => { + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }) + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + } + } + }) + await session.reconnect(deps.mockConn) + const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length + + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 5, + sourceEndSu: 8 + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount + 1) + expect(closeSshPtyOutputGeneration).toHaveBeenLastCalledWith( + 23, + 'ssh_source_frame_stale_or_non_contiguous' + ) + }) + + it('drops late frames from a token after its cancellation proof is validated', async () => { + const targetId = 'late-after-cancel' + muxRequestMock.mockImplementation(async (method) => + method === 'pty.cancelDelivery' ? { canceled: true, sentEndSu: 8, creditedEndSu: 4 } : [] + ) + const { session, deps } = await prepareRecovery(targetId) + attachForReconnectMock.mockImplementation(async () => { + queueMicrotask(() => { + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 5, + sourceEndSu: 8 + }) + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }) + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + } + } + }) + await session.reconnect(deps.mockConn) + const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length + + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 8, + sourceEndSu: 12 + }) + + expect(acceptOutputDataMock).not.toHaveBeenCalled() + expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'detached') + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + }) + + it('rejects cancellation proof below capacity-rejected and later private frames', async () => { + const targetId = 'capacity-watermark' + muxRequestMock.mockImplementation(async (method) => + method === 'pty.cancelDelivery' ? { canceled: true, sentEndSu: 8, creditedEndSu: 4 } : [] + ) + const { session, deps } = await prepareRecovery(targetId) + const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() } + const sourceActivationLease = { + commit: vi.fn(), + rollback: vi.fn(), + transferToRecovery: vi.fn((sink: (payload: unknown) => void) => { + sink({ + id: `ssh:${targetId}@@pty-1`, + data: 'x'.repeat(2 * 1024 * 1024 + 1), + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: 4, + source: { + relayPtyId: 'pty-1', + spanId: 'new-token:4:8', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu: 4, + sourceEndSu: 8 + } + }) + sink({ + id: `ssh:${targetId}@@pty-1`, + data: 'later', + providerGeneration: 23, + ptyIncarnation: 'incarnation-1', + sequenceChars: 4, + source: { + relayPtyId: 'pty-1', + spanId: 'new-token:8:12', + clientGeneration: 2, + ownerGeneration: 2, + deliveryToken: 'new-token', + sourceStartSu: 8, + sourceEndSu: 12 + } + }) + return recoveryActivationLease + }) + } + attachForReconnectMock.mockResolvedValue({ + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 12 + }, + sourceActivationLease + }) + + await session.reconnect(deps.mockConn) + + expect(muxRequestMock).toHaveBeenCalledWith( + 'pty.cancelDelivery', + expect.objectContaining({ deliveryToken: 'new-token' }) + ) + expect(vi.mocked(closeSshPtyOutputGeneration).mock.calls).toContainEqual([ + 23, + 'ssh_source_recovery_cancellation_failed' + ]) + expect(applySshPtySourceRecoveryCancellationProof).not.toHaveBeenCalled() + expect(recoveryActivationLease.commit).not.toHaveBeenCalled() + expect(recoveryActivationLease.retire).toHaveBeenCalledOnce() + }) + + it.each([ + ['publication', undefined], + ['proof', { canceled: true, sentEndSu: 8, creditedEndSu: 4 }], + ['negative-end', { canceled: true, sentEndSu: -1, creditedEndSu: 4 }], + ['credited-ahead', { canceled: true, sentEndSu: 8, creditedEndSu: 9 }], + ['checkpoint-mismatch', { canceled: true, sentEndSu: 8, creditedEndSu: 3 }], + ['sent-before-private-end', { canceled: true, sentEndSu: 7, creditedEndSu: 4 }] + ] as const)( + 'closes one provider generation when recovery cancellation %s rejects', + async (failure, cancellationResult) => { + const targetId = `cancel-${failure}-failure` + muxRequestMock.mockImplementation(async (method) => { + if (method !== 'pty.cancelDelivery') { + return [] + } + if (failure === 'publication') { + throw new Error('cancel publication failed') + } + return cancellationResult + }) + if (failure === 'proof') { + vi.mocked(applySshPtySourceRecoveryCancellationProof).mockImplementation(() => { + throw new Error('cancel proof rejected') + }) + } + const { session, deps } = await prepareRecovery(targetId) + const onRelayLost = vi.fn() + const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() } + const activationLease = { + commit: vi.fn(), + rollback: vi.fn(), + transferToRecovery: vi.fn(() => recoveryActivationLease) + } + session.setOnRelayLost(onRelayLost) + let cleanupCountsBeforeFailure: + | { + generation: number + mux: number + provider: number + ack: number + cancellation: number + unregister: number + } + | undefined + attachForReconnectMock.mockImplementation(async () => { + cleanupCountsBeforeFailure = { + generation: vi.mocked(closeSshPtyOutputGeneration).mock.calls.length, + mux: muxDisposeMock.mock.calls.length, + provider: ptyProviderDisposeMock.mock.calls.length, + ack: sourceAckCleanupMock.mock.calls.length, + cancellation: sourceCancellationCleanupMock.mock.calls.length, + unregister: vi.mocked(unregisterSshPtyProvider).mock.calls.length + } + queueMicrotask(() => { + emitSourceFrame({ + targetId, + token: 'new-token', + clientGeneration: 2, + ownerGeneration: 2, + sourceStartSu: 5, + sourceEndSu: 8 + }) + }) + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: 2, + ownerGeneration: 2, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'new-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }, + sourceActivationLease: activationLease + } + }) + + await session.reconnect(deps.mockConn) + + expect(cleanupCountsBeforeFailure).toBeDefined() + const before = cleanupCountsBeforeFailure! + expect(vi.mocked(closeSshPtyOutputGeneration).mock.calls).toHaveLength(before.generation + 1) + expect(closeSshPtyOutputGeneration).toHaveBeenLastCalledWith( + 23, + 'ssh_source_recovery_cancellation_failed' + ) + expect(muxDisposeMock.mock.calls).toHaveLength(before.mux + 1) + expect(muxDisposeMock).toHaveBeenLastCalledWith('connection_lost') + expect(ptyProviderDisposeMock.mock.calls).toHaveLength(before.provider + 1) + expect(sourceAckCleanupMock.mock.calls).toHaveLength(before.ack + 1) + expect(sourceCancellationCleanupMock.mock.calls).toHaveLength(before.cancellation + 1) + expect(vi.mocked(unregisterSshPtyProvider).mock.calls).toHaveLength(before.unregister + 1) + expect(unregisterSshPtyProvider).toHaveBeenLastCalledWith(targetId) + expect(activationLease.transferToRecovery).toHaveBeenCalledOnce() + expect(activationLease.commit).not.toHaveBeenCalled() + expect(activationLease.rollback).not.toHaveBeenCalled() + expect(recoveryActivationLease.commit).not.toHaveBeenCalled() + expect(recoveryActivationLease.retire).toHaveBeenCalledOnce() + expect(onRelayLost).toHaveBeenCalledOnce() + expect(session.getState()).toBe('reconnecting') + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalled() + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith( + 'pty:exit', + expect.anything() + ) + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + expect(vi.mocked(closeSshPtyOutputGeneration).mock.calls).not.toContainEqual([ + 99, + expect.anything() + ]) + expect(unregisterSshPtyProvider).not.toHaveBeenCalledWith('unrelated-target') + } + ) + + it('keeps a stale overlapping recovery from canceling or mutating its replacement', async () => { + const targetId = 'overlapping-recovery' + const { session, deps } = await prepareRecovery(targetId) + const staleRecoveryLease = { commit: vi.fn(), retire: vi.fn() } + const replacementRecoveryLease = { commit: vi.fn(), retire: vi.fn() } + const staleLease = { + commit: vi.fn(), + rollback: vi.fn(), + transferToRecovery: vi.fn(() => staleRecoveryLease) + } + const replacementLease = { + commit: vi.fn(), + rollback: vi.fn(), + transferToRecovery: vi.fn(() => replacementRecoveryLease) + } + attachForReconnectMock.mockImplementation(async () => { + const ownerGeneration = openConsumerSessionMock.mock.calls.length + if (ownerGeneration === 3) { + queueMicrotask(() => { + const complete = onNotificationByMethodMock.mock.calls.findLast( + ([method]) => method === 'pty.recoveryComplete' + )?.[1] as ((params: Record) => void) | undefined + complete?.({ + id: 'pty-1', + clientGeneration: 3, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'replacement-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }) + }) + } + return { + incarnationId: 'incarnation-1', + sourceRecovery: { + status: 'pending', + clientGeneration: ownerGeneration, + ownerGeneration, + ptyIncarnation: 'incarnation-1', + deliveryToken: ownerGeneration === 2 ? 'stale-token' : 'replacement-token', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }, + sourceActivationLease: ownerGeneration === 2 ? staleLease : replacementLease + } + }) + + const staleReconnect = session.reconnect(deps.mockConn) + await vi.waitFor(() => expect(attachForReconnectMock).toHaveBeenCalledTimes(1)) + await Promise.resolve() + const replacementReconnect = session.reconnect(deps.mockConn) + await Promise.all([staleReconnect, replacementReconnect]) + + const recoveryRequests = attachForReconnectMock.mock.calls.map((call) => call[2]) + expect(recoveryRequests).toHaveLength(2) + expect(recoveryRequests[1]).toMatchObject({ + status: 'checkpoint', + deliveryToken: 'old-token', + acceptedSourceEndSu: 4 + }) + expect(muxRequestMock.mock.calls.filter(([method]) => method === 'pty.cancelDelivery')).toEqual( + [] + ) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledTimes(1) + expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'attached') + expect(setPtyOwnership).toHaveBeenCalledTimes(1) + expect(staleLease.transferToRecovery).toHaveBeenCalledOnce() + expect(staleLease.commit).not.toHaveBeenCalled() + expect(staleLease.rollback).not.toHaveBeenCalled() + expect(staleRecoveryLease.commit).not.toHaveBeenCalled() + expect(staleRecoveryLease.retire).toHaveBeenCalledOnce() + expect(replacementLease.transferToRecovery).toHaveBeenCalledOnce() + expect(replacementLease.commit).not.toHaveBeenCalled() + expect(replacementLease.rollback).not.toHaveBeenCalled() + expect(replacementRecoveryLease.commit).toHaveBeenCalledOnce() + expect(replacementRecoveryLease.retire).not.toHaveBeenCalled() + expect(clearProviderPtyState).not.toHaveBeenCalled() + expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled() + expect(deletePtyOwnership).not.toHaveBeenCalled() + expect(deps.mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:exit', expect.anything()) + expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown') + expect(session.getState()).toBe('ready') + }) +}) diff --git a/src/main/ssh/ssh-relay-session-terminal-error.test.ts b/src/main/ssh/ssh-relay-session-terminal-error.test.ts index ce9317c6660..52be21dcc2e 100644 --- a/src/main/ssh/ssh-relay-session-terminal-error.test.ts +++ b/src/main/ssh/ssh-relay-session-terminal-error.test.ts @@ -15,12 +15,22 @@ vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + openSshPtyConsumerSession: vi.fn(async (_mux, options) => ({ + clientInstanceId: options.clientInstanceId, + clientGeneration: 1, + ownerGeneration: 1, + ownerLease: 'test-owner-lease' + })) +})) + vi.mock('./ssh-channel-multiplexer', () => { return { SshChannelMultiplexer: class MockSshChannelMultiplexer { notify = vi.fn() request = vi.fn().mockResolvedValue([]) onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) onRequest = vi.fn().mockReturnValue(() => {}) onDispose = vi.fn().mockReturnValue(() => {}) dispose = vi.fn() diff --git a/src/main/ssh/ssh-relay-session-test-fixtures.ts b/src/main/ssh/ssh-relay-session-test-fixtures.ts index 679758d5e11..b319140c0cf 100644 --- a/src/main/ssh/ssh-relay-session-test-fixtures.ts +++ b/src/main/ssh/ssh-relay-session-test-fixtures.ts @@ -1,5 +1,6 @@ import { vi, type Mock } from 'vitest' import type { BrowserWindow } from 'electron' +import { PtyConsumerSession } from '../../shared/pty-consumer-session' import type { SshConnection } from './ssh-connection' import type { Store } from '../persistence' import type { SshPortForwardManager } from './ssh-port-forward' @@ -47,3 +48,39 @@ export function mockDeploySuccess(): void { platform: 'linux-x64' }) } + +export function createMismatchedOwnerRecoveryError(): unknown { + const stateMachine = new PtyConsumerSession({ + serverBuildId: 'test-relay-build', + createLease: () => 'retained-owner-lease' + }) + const owner = stateMachine.admit( + { clientInstanceId: 'retained-client', requestedRole: 'session-owner' }, + { + connectionId: 'retained-connection', + principal: 'retained-principal', + authenticated: true, + allowSessionOwner: true + } + ) + owner.commitPublication() + stateMachine.close('retained-connection') + try { + stateMachine.admit( + { + clientInstanceId: 'retained-client', + requestedRole: 'session-owner', + resume: { ownerGeneration: 1, ownerLease: 'retained-owner-lease' } + }, + { + connectionId: 'stale-connection', + principal: 'stale-principal', + authenticated: true, + allowSessionOwner: true + } + ) + } catch (error) { + return error + } + throw new Error('Expected mismatched owner recovery to fail') +} diff --git a/src/main/ssh/ssh-relay-session.test.ts b/src/main/ssh/ssh-relay-session.test.ts index 8fe7a292bcf..40b38a75b52 100644 --- a/src/main/ssh/ssh-relay-session.test.ts +++ b/src/main/ssh/ssh-relay-session.test.ts @@ -8,12 +8,39 @@ import { import { SSH_RELAY_CONFIGURE_GRACE_TIME_METHOD } from '../../shared/ssh-types' import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' -const { muxRequestMock } = vi.hoisted(() => ({ muxRequestMock: vi.fn() })) +const { acceptOutputDataMock, muxRequestMock, openConsumerSessionMock, pauseAdapterMock } = + vi.hoisted(() => ({ + acceptOutputDataMock: vi.fn().mockResolvedValue(undefined), + muxRequestMock: vi.fn(), + openConsumerSessionMock: vi.fn(), + pauseAdapterMock: vi.fn() + })) vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) +vi.mock('./ssh-pty-consumer-session', () => ({ + SSH_PTY_SOURCE_WINDOW_SU: 256 * 1024, + openSshPtyConsumerSession: openConsumerSessionMock +})) + +vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({ + acceptSshPtyOutputData: acceptOutputDataMock, + acceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined), + allocateSshPtyProviderGeneration: vi.fn(() => 41), + beginSshPtyOutputGenerationMigration: vi.fn(() => ({ + byPty: new Map(), + completion: Promise.resolve() + })), + closeSshPtyOutputGeneration: vi.fn(), + getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []), + applySshPtySourceCancellationProof: vi.fn(() => true), + applySshPtySourceRecoveryCancellationProof: vi.fn(() => true), + installSshPtySourceAckPublisher: vi.fn(() => () => {}), + installSshPtySourceCancellationPublisher: vi.fn(() => () => {}) +})) + vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) @@ -22,8 +49,10 @@ vi.mock('./ssh-channel-multiplexer', () => { return { SshChannelMultiplexer: class MockSshChannelMultiplexer { notify = vi.fn() + notifyWithSettlement = vi.fn() request = muxRequestMock onNotification = vi.fn().mockReturnValue(() => {}) + onNotificationByMethod = vi.fn().mockReturnValue(() => {}) onRequest = vi.fn().mockReturnValue(() => {}) onDispose = vi.fn().mockReturnValue(() => {}) dispose = vi.fn() @@ -43,6 +72,7 @@ vi.mock('../providers/ssh-pty-provider', () => ({ onExit = vi.fn().mockReturnValue(() => {}) attach = vi.fn().mockResolvedValue(undefined) attachForReconnect = vi.fn().mockResolvedValue({}) + setPtyDeliveryPauseAdapter = pauseAdapterMock dispose = vi.fn() } })) @@ -86,12 +116,6 @@ vi.mock('../providers/ssh-git-dispatch', () => ({ })) const { deployAndLaunchRelay } = await import('./ssh-relay-deploy') -// Why: the hidden-delivery gate module is intentionally real (pure state, no -// electron deps) so the SSH parity tests exercise the same gate main uses. -const { markHiddenRendererPty, setRendererPtyDeliveryInterest } = - await import('../ipc/pty-hidden-delivery-gate') -const { _resetHiddenRendererPtyDeliveryGateForTest } = - await import('../ipc/pty-hidden-delivery-gate') const { execCommand } = await import('./ssh-relay-deploy-helpers') const { getRemoteHostPlatform } = await import('./ssh-remote-platform') const { @@ -110,20 +134,21 @@ const { registerSshGitProvider, unregisterSshGitProvider } = describe('SshRelaySession', () => { beforeEach(() => { vi.clearAllMocks() + openConsumerSessionMock.mockImplementation(async (_mux, options) => ({ + mode: 'legacy-fallback', + clientInstanceId: options.clientInstanceId, + serverBuildId: 'test-relay-build' + })) delete process.env.ORCA_FEATURE_REMOTE_AGENT_HOOKS muxRequestMock.mockReset() muxRequestMock.mockResolvedValue([]) mockDeploySuccess() vi.mocked(getPtyIdsForConnection).mockReturnValue([]) - _resetHiddenRendererPtyDeliveryGateForTest() }) - it('drops hidden-gated PTY data after runtime ingestion with one restore marker', async () => { + it('hands each PTY data event exactly once to the bounded main intake', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() - const runtime = { - onPtyData: vi.fn(() => 99), - onPtyExit: vi.fn() - } + const runtime = { onPtyData: vi.fn(), onPtyExit: vi.fn() } const session = new SshRelaySession( 'target-1', getMainWindow, @@ -138,75 +163,28 @@ describe('SshRelaySession', () => { const onData = ptyProvider.onData.mock.calls[0]?.[0] as (payload: { id: string data: string + providerGeneration: number + ptyIncarnation: string }) => void - markHiddenRendererPty('ssh-pty-1') - onData({ id: 'ssh-pty-1', data: 'hidden ssh output' }) - - // Runtime ingestion still ran; renderer delivery shrank to one marker. - expect(runtime.onPtyData).toHaveBeenCalledWith( - 'ssh-pty-1', - 'hidden ssh output', - expect.any(Number), - 'hidden ssh output'.length, - undefined - ) - expect(mockWindow.webContents.send).toHaveBeenCalledTimes(1) - // Why out-of-band: an in-band empty pty:data sentinel is ambiguous with - // chunks fully consumed by renderer OSC-9999 stripping. - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:modelRestoreNeeded', { + onData({ id: 'ssh-pty-1', - reason: 'hidden-drop', - markerSeq: 99 + data: 'ssh output', + providerGeneration: 41, + ptyIncarnation: 'incarnation-1' }) - onData({ id: 'ssh-pty-1', data: 'more hidden ssh output' }) - expect(mockWindow.webContents.send).toHaveBeenCalledTimes(1) - - // Delivery interest (renderer sidecars) suppresses the gate — parity with - // the local path in ipc/pty.ts. - setRendererPtyDeliveryInterest('ssh-pty-1', true) - onData({ id: 'ssh-pty-1', data: 'sidecar ssh bytes' }) - expect(mockWindow.webContents.send).toHaveBeenLastCalledWith('pty:data', { + expect(acceptOutputDataMock).toHaveBeenCalledTimes(1) + expect(acceptOutputDataMock).toHaveBeenCalledWith({ id: 'ssh-pty-1', - data: 'sidecar ssh bytes', - seq: 99, - rawLength: 'sidecar ssh bytes'.length - }) - - // Non-hidden PTYs are unaffected. - onData({ id: 'ssh-pty-2', data: 'visible ssh output' }) - expect(mockWindow.webContents.send).toHaveBeenLastCalledWith('pty:data', { - id: 'ssh-pty-2', - data: 'visible ssh output', - seq: 99, - rawLength: 'visible ssh output'.length - }) - }) - - it('keeps hidden SSH delivery when the gate kill switch is off', async () => { - const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() - ;(mockStore as unknown as { getSettings: () => unknown }).getSettings = vi.fn(() => ({ - terminalHiddenDeliveryGate: false - })) - const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) - await session.establish(mockConn) - const ptyProvider = vi.mocked(registerSshPtyProvider).mock.calls[0]?.[1] as unknown as { - onData: ReturnType - } - const onData = ptyProvider.onData.mock.calls[0]?.[0] as (payload: { - id: string - data: string - }) => void - - markHiddenRendererPty('ssh-pty-1') - onData({ id: 'ssh-pty-1', data: 'still delivered' }) - - expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:data', { - id: 'ssh-pty-1', - data: 'still delivered', - rawLength: 'still delivered'.length + data: 'ssh output', + providerGeneration: 41, + ptyIncarnation: 'incarnation-1', + rawLength: 'ssh output'.length, + transformed: false }) + expect(runtime.onPtyData).not.toHaveBeenCalled() + expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:data', expect.anything()) }) it('starts in idle state', () => { @@ -501,7 +479,7 @@ describe('SshRelaySession', () => { }) }) - it('drops identical reconnect replay payloads inside one reconnect burst', async () => { + it('does not wall-clock dedupe identical replay payloads from distinct reconnects', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) await session.establish(mockConn) @@ -523,7 +501,7 @@ describe('SshRelaySession', () => { .mocked(mockWindow.webContents.send) .mock.calls.filter(([channel]) => channel === 'pty:replay') expect(mockAttach).toHaveBeenCalledTimes(2) - expect(replaySends).toHaveLength(1) + expect(replaySends).toHaveLength(2) }) it('establish re-attaches owned PTYs after explicit disconnect', async () => { @@ -747,7 +725,7 @@ describe('SshRelaySession', () => { }) }) - it('routes transient reattach failures through relay-lost retry handling', async () => { + it('retries transient reattach failure without tearing down provider registration', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) const onRelayLost = vi.fn() @@ -763,11 +741,17 @@ describe('SshRelaySession', () => { dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-live']) + const random = vi.spyOn(Math, 'random').mockReturnValue(0) - await session.reconnect(mockConn) + try { + await session.reconnect(mockConn) + } finally { + random.mockRestore() + } - expect(mockAttach).toHaveBeenCalledWith('pty-live') - expect(onRelayLost).toHaveBeenCalledWith('target-1') + expect(mockAttach).toHaveBeenCalledTimes(2) + expect(onRelayLost).not.toHaveBeenCalled() + expect(session.getState()).toBe('ready') expect(mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith( 'target-1', 'pty-live', diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index 392a422e138..0af4f59969b 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -1,15 +1,17 @@ /* oxlint-disable max-lines */ // Why: single authority for all relay lifecycle state per SSH target (previously scattered across module Maps/Sets with duplicated paths). -import type { BrowserWindow } from 'electron' import { randomUUID } from 'node:crypto' +import type { BrowserWindow } from 'electron' import { deployAndLaunchRelay } from './ssh-relay-deploy' import { execCommand } from './ssh-relay-deploy-helpers' import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error' import type { RelayVersionMismatchError } from './ssh-relay-version-mismatch-error' import { SshChannelMultiplexer } from './ssh-channel-multiplexer' import { SshPtyProvider } from '../providers/ssh-pty-provider' -import type { SshPtyExitCallback } from '../providers/ssh-pty-provider-contract' +import type { SshPtyAttachResult } from '../providers/ssh-pty-session-reattach' +import type { SshPtyDataCallback, SshPtyExitCallback } from '../providers/ssh-pty-provider-contract' +import type { SshPtyRecoveryActivationLease } from '../providers/ssh-pty-notification-routing' import { isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors' import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id' import { SshFilesystemProvider } from '../providers/ssh-filesystem-provider' @@ -38,10 +40,19 @@ import { isCurrentPtyExit } from '../ipc/pty' import { - recordHiddenRendererPtyDataDrop, - shouldDropHiddenRendererPtyData -} from '../ipc/pty-hidden-delivery-gate' -import type { PtyModelRestoreNeededEvent } from '../../shared/pty-model-restore-marker' + acceptSshPtyOutputData, + acceptSshPtyOutputExit, + allocateSshPtyProviderGeneration, + applySshPtySourceCancellationProof, + applySshPtySourceRecoveryCancellationProof, + beginSshPtyOutputGenerationMigration, + closeSshPtyOutputGeneration, + getSshPtyAcceptedSourceCheckpoints, + installSshPtySourceAckPublisher, + installSshPtySourceCancellationPublisher +} from '../ipc/ssh-pty-output-intake-registry' +import type { SshPtyAcceptedSourceCheckpoint } from '../ipc/ssh-pty-output-source-obligations' +import type { SshPtyOutputMigrationResult } from '../ipc/ssh-pty-output-model-migration' import { registerSshFilesystemProvider, unregisterSshFilesystemProvider, @@ -65,6 +76,8 @@ import { } from '../../shared/ssh-types' import type { Store } from '../persistence' import type { OrcaRuntimeService } from '../runtime/orca-runtime' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract' +import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session' import { runRemoteOrcaCli } from './ssh-remote-orca-cli' import { acknowledgeRemoteOrcaCliPostOutput, @@ -73,11 +86,47 @@ import { import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' import { isValidTerminalTabId } from '../../shared/terminal-tab-id' +import { + openSshPtyConsumerSession, + type SshPtyConsumerOwnerState, + type SshPtyConsumerSessionState +} from './ssh-pty-consumer-session' +import type { + PtySourceRecoveryComplete, + PtySourceRecoveryPending, + PtySourceRecoveryRequest +} from '../../shared/pty-source-recovery-contract' +import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget' +import { SshPtyRetiredSourceDeliveries } from './ssh-pty-retired-source-deliveries' export type RelaySessionState = 'idle' | 'deploying' | 'ready' | 'reconnecting' | 'disposed' type SshPtyExitPayload = Parameters[0] -type PendingPtyReattach = { exits: SshPtyExitPayload[] } +type SshPtyDataPayload = Parameters[0] +type SshPtyLease = ReturnType[number] +const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8 +const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000 +const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50 +const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200 +const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed' +type PendingPtyReattach = { + mux: SshChannelMultiplexer + providerGeneration: number + retentionKey: string + exits: SshPtyExitPayload[] + queuedData: SshPtyDataPayload[] + recoveryData: SshPtyDataPayload[] + liveData: SshPtyDataPayload[] + recovery?: PtySourceRecoveryPending + recoveryComplete?: PtySourceRecoveryComplete + nextRecoverySourceSu?: number + highestRecoverySourceEndSu?: number + replacementDeliveryToken?: string + restoreRequired?: string + recoveryWaiters: Set<() => void> + livePassthrough: boolean + activated: boolean +} type RemoteCliBridgeEnv = { remoteHome: string @@ -85,6 +134,7 @@ type RemoteCliBridgeEnv = { relayDir: string nodePath: string sockPath: string + credentialFile?: string hostPlatform: RemoteHostPlatform pathDelimiter?: ':' | ';' } @@ -110,9 +160,49 @@ function expectedIdentityForLease(lease: { } } -type ForwardedReplayFingerprint = { - fingerprint: string - deliveredAt: number +function parseRecoveryComplete(params: Record): PtySourceRecoveryComplete | null { + if ( + typeof params.id !== 'string' || + typeof params.deliveryToken !== 'string' || + params.deliveryToken.length === 0 || + typeof params.ptyIncarnation !== 'string' || + params.ptyIncarnation.length === 0 || + !positiveSafeInteger(params.clientGeneration) || + !positiveSafeInteger(params.ownerGeneration) || + !nonNegativeSafeInteger(params.checkpointSourceEndSu) || + !nonNegativeSafeInteger(params.recoveryEndSu) || + Number(params.recoveryEndSu) < Number(params.checkpointSourceEndSu) + ) { + return null + } + return Object.freeze({ + id: params.id, + deliveryToken: params.deliveryToken, + ptyIncarnation: params.ptyIncarnation, + clientGeneration: Number(params.clientGeneration), + ownerGeneration: Number(params.ownerGeneration), + checkpointSourceEndSu: Number(params.checkpointSourceEndSu), + recoveryEndSu: Number(params.recoveryEndSu) + }) +} + +function positiveSafeInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) > 0 +} + +function nonNegativeSafeInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) >= 0 +} + +function sourceRecoveryCancellationError(cause: unknown): Error { + return Object.assign(new Error(SSH_SOURCE_RECOVERY_CANCELLATION_FAILED), { + code: SSH_SOURCE_RECOVERY_CANCELLATION_FAILED, + cause + }) +} + +function isSourceRecoveryCancellationError(error: unknown): boolean { + return (error as { code?: unknown })?.code === SSH_SOURCE_RECOVERY_CANCELLATION_FAILED } export type SshRelayAiVaultHostInfo = { @@ -122,9 +212,6 @@ export type SshRelayAiVaultHostInfo = { hostPlatform: RemoteHostPlatform } -const RECONNECT_REPLAY_DUPLICATE_WINDOW_MS = 1000 -const REPLAY_FINGERPRINT_EDGE_CHARS = 128 - function normalizeRelayGracePeriodSeconds(graceTimeSeconds: number | undefined): number { const raw = graceTimeSeconds ?? DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS const requested = Number.isFinite(raw) ? Math.floor(raw) : DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS @@ -136,6 +223,33 @@ function normalizeRelayGracePeriodSeconds(graceTimeSeconds: number | undefined): ) } +type PtyConsumerRecovery = { + clientInstanceId: string + detached: boolean + serverBuildId?: string + owner?: SshPtyConsumerOwnerState + checkpointsByAppPtyId: Map + modelMigrationsByAppPtyId: Map> +} + +const ptyConsumerRecoveryByTarget = new Map() + +function ptyConsumerRecoveryForTarget(targetId: string): PtyConsumerRecovery { + const current = ptyConsumerRecoveryByTarget.get(targetId) + if (current?.detached) { + current.detached = false + return current + } + const created = { + clientInstanceId: randomUUID(), + detached: false, + checkpointsByAppPtyId: new Map(), + modelMigrationsByAppPtyId: new Map>() + } + ptyConsumerRecoveryByTarget.set(targetId, created) + return created +} + export class SshRelaySession { private _state: RelaySessionState = 'idle' private mux: SshChannelMultiplexer | null = null @@ -154,8 +268,25 @@ export class SshRelaySession { private currentConnection: SshConnection | null = null private hostPlatform: RemoteHostPlatform | null = null private remoteCliBridgeEnv: RemoteCliBridgeEnv | null = null - private forwardedReattachReplayByPty = new Map() private pendingPtyReattaches = new Map() + private readonly ptyRecoveryRetention = new SshPtyRecoveryRetentionBudget() + private activePtyProviderGeneration: number | null = null + private sourceAckPublisherCleanup: (() => void) | null = null + private sourceCancellationPublisherCleanup: (() => void) | null = null + private ptyRecoveryNotificationCleanups: (() => void)[] = [] + private readonly sourceIdentityByRelayPtyId = new Map< + string, + Readonly<{ + deliveryToken: string + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + nextSourceSu?: number + }> + >() + private readonly retiredSourceDeliveries = new SshPtyRetiredSourceDeliveries() + private readonly ptyConsumerClientInstanceId: string + private ptyConsumerSessionState: SshPtyConsumerSessionState | null = null private activeCompatibilityAttachmentIds = new Set() constructor( @@ -169,7 +300,9 @@ export class SshRelaySession { ports: DetectedPort[], platform: string ) => void - ) {} + ) { + this.ptyConsumerClientInstanceId = ptyConsumerRecoveryForTarget(targetId).clientInstanceId + } refreshEnvironment( getMainWindow: () => BrowserWindow | null, @@ -255,8 +388,16 @@ export class SshRelaySession { this.currentConnection = conn try { - const { transport, remoteHome, remoteRelayDir, nodePath, sockPath, hostPlatform } = - await deployAndLaunchRelay(conn, undefined, graceTimeSeconds, this.targetId) + const { + transport, + serverBuildId, + remoteHome, + remoteRelayDir, + nodePath, + sockPath, + credentialFile, + hostPlatform + } = await deployAndLaunchRelay(conn, undefined, graceTimeSeconds, this.targetId) this.hostPlatform = hostPlatform ?? null this.remoteCliBridgeEnv = remoteHome && remoteRelayDir && nodePath && sockPath && hostPlatform @@ -266,6 +407,7 @@ export class SshRelaySession { relayDir: remoteRelayDir, nodePath, sockPath, + ...(credentialFile ? { credentialFile } : {}), hostPlatform, pathDelimiter: hostPlatform.pathDelimiter } @@ -282,7 +424,9 @@ export class SshRelaySession { this.mux = mux const ownsAttempt = (): boolean => this.mux === mux && !this.isDisposed() - // Why: round-trip the relay before registering providers so a closed --connect bridge fails fast instead of leaving a 'ready' session on a dead mux. + this.ptyConsumerSessionState = await this.openPtyConsumerSession(mux, serverBuildId) + this.rememberPtyConsumerRecovery(serverBuildId) + await mux.request('session.resolveHome', { path: '~' }) if (!ownsAttempt()) { if (!mux.isDisposed()) { @@ -311,7 +455,7 @@ export class SshRelaySession { } // Why: explicit disconnect keeps PTY ownership, so a later manual connect must reattach those remote PTYs. - await this.reattachKnownPtys(ownsAttempt) + await this.reattachKnownPtys(mux, ownsAttempt) if (!ownsAttempt()) { throw new Error('Session disposed during establish') @@ -325,7 +469,12 @@ export class SshRelaySession { } catch (err) { // Why: registerProviders can throw with a live mux and partial registration — tear everything down so a retry starts clean. if (!this.isDisposed()) { - this.teardownProviders('connection_lost') + this.teardownProviders( + 'connection_lost', + isSourceRecoveryCancellationError(err) + ? SSH_SOURCE_RECOVERY_CANCELLATION_FAILED + : 'connection_lost' + ) this._state = 'idle' } // Why: a version mismatch on first connect is terminal (deployed binary vs. a still-running legacy daemon); notify the callback but still rethrow. @@ -361,8 +510,16 @@ export class SshRelaySession { this.teardownProviders('connection_lost') try { - const { transport, remoteHome, remoteRelayDir, nodePath, sockPath, hostPlatform } = - await deployAndLaunchRelay(conn, undefined, graceTimeSeconds, this.targetId) + const { + transport, + serverBuildId, + remoteHome, + remoteRelayDir, + nodePath, + sockPath, + credentialFile, + hostPlatform + } = await deployAndLaunchRelay(conn, undefined, graceTimeSeconds, this.targetId) this.hostPlatform = hostPlatform ?? null this.remoteCliBridgeEnv = remoteHome && remoteRelayDir && nodePath && sockPath && hostPlatform @@ -372,6 +529,7 @@ export class SshRelaySession { relayDir: remoteRelayDir, nodePath, sockPath, + ...(credentialFile ? { credentialFile } : {}), hostPlatform, pathDelimiter: hostPlatform.pathDelimiter } @@ -392,7 +550,15 @@ export class SshRelaySession { !abortController.signal.aborted && !this.isDisposed() - // Why: same health check as establish() — round-trip the relay before registering providers so a dead --connect bridge fails fast. + this.ptyConsumerSessionState = await this.openPtyConsumerSession(mux, serverBuildId) + this.rememberPtyConsumerRecovery(serverBuildId) + if (!ownsAttempt()) { + if (!mux.isDisposed()) { + mux.dispose() + } + return + } + await mux.request('session.resolveHome', { path: '~' }) if (!ownsAttempt()) { if (!mux.isDisposed()) { @@ -424,7 +590,7 @@ export class SshRelaySession { return } - await this.reattachKnownPtys(ownsAttempt) + await this.reattachKnownPtys(mux, ownsAttempt) if (!ownsAttempt()) { return @@ -438,7 +604,12 @@ export class SshRelaySession { } catch (err) { // Why: tear down a partially-registered mux so its keepalive/timeout timers don't keep running on a half-initialized session. if (this.abortController === abortController && !this.isDisposed()) { - this.teardownProviders('connection_lost') + this.teardownProviders( + 'connection_lost', + isSourceRecoveryCancellationError(err) + ? SSH_SOURCE_RECOVERY_CANCELLATION_FAILED + : 'connection_lost' + ) } // Why: version-mismatch is terminal — fire the typed callback and drop out of 'reconnecting' since backoff retry can't reconcile it. if (isRelayVersionMismatchError(err)) { @@ -479,6 +650,7 @@ export class SshRelaySession { this.store.markSshRemotePtyLeases(this.targetId, 'terminated') this.currentConnection = null this._state = 'disposed' + ptyConsumerRecoveryByTarget.delete(this.targetId) } detach(): void { @@ -493,6 +665,10 @@ export class SshRelaySession { this.store.markSshRemotePtyLeases(this.targetId, 'detached') this.currentConnection = null this._state = 'disposed' + const recovery = ptyConsumerRecoveryByTarget.get(this.targetId) + if (recovery?.clientInstanceId === this.ptyConsumerClientInstanceId) { + recovery.detached = true + } } // ── Private ─────────────────────────────────────────────────────── @@ -547,8 +723,74 @@ export class SshRelaySession { this.wireUpRemoteOrcaCli(mux, connectionIncarnation) - const ptyProvider = new SshPtyProvider(this.targetId, mux, this.remoteCliBridgeEnv ?? undefined) + const providerGeneration = allocateSshPtyProviderGeneration() + const ptyProvider = new SshPtyProvider( + this.targetId, + mux, + this.remoteCliBridgeEnv ?? undefined, + providerGeneration + ) + const consumerOwnerState = this.negotiatedPtyConsumerOwner() + if (consumerOwnerState) { + ptyProvider.setPtyDeliveryPauseAdapter?.(({ id, providerGeneration: generation, paused }) => { + if ( + generation !== providerGeneration || + this.activePtyProviderGeneration !== providerGeneration || + this.mux !== mux + ) { + return + } + const sourceIdentity = this.sourceIdentityByRelayPtyId.get(id) + if (consumerOwnerState.outputFlowControl && !sourceIdentity) { + return + } + mux.notify('pty.setDeliveryPaused', { + id, + paused, + clientGeneration: consumerOwnerState.clientGeneration, + ownerGeneration: consumerOwnerState.ownerGeneration, + ...(sourceIdentity ? { deliveryToken: sourceIdentity.deliveryToken } : {}) + }) + }) + } + this.sourceAckPublisherCleanup?.() + this.sourceAckPublisherCleanup = null + this.sourceCancellationPublisherCleanup?.() + this.sourceCancellationPublisherCleanup = null + if (consumerOwnerState?.outputFlowControl) { + this.sourceAckPublisherCleanup = installSshPtySourceAckPublisher( + providerGeneration, + (batch, onSettled) => + mux.notifyWithSettlement( + 'pty.ackData', + batch as unknown as Record, + onSettled + ) + ) + this.sourceCancellationPublisherCleanup = installSshPtySourceCancellationPublisher( + providerGeneration, + async (request) => { + const result = (await mux.request('pty.cancelDelivery', { + ...request, + id: toRelaySshPtyId(this.targetId, request.id) + })) as Record + if ( + result.canceled !== true || + !Number.isSafeInteger(result.sentEndSu) || + !Number.isSafeInteger(result.creditedEndSu) + ) { + throw new Error('ssh_source_cancellation_proof_invalid') + } + return { + sentEndSu: result.sentEndSu as number, + creditedEndSu: result.creditedEndSu as number + } + } + ) + } + this.activePtyProviderGeneration = providerGeneration registerSshPtyProvider(this.targetId, ptyProvider) + this.installPtyRecoveryNotifications(mux) const connection = this.requireReadyConnection() const createSftp = @@ -588,12 +830,93 @@ export class SshRelaySession { ) registerSshGitProvider(this.targetId, gitProvider) - this.wireUpPtyEvents(ptyProvider) + this.wireUpPtyEvents(ptyProvider, mux, providerGeneration) this.wireUpAgentHookEvents(mux) this.wireUpRemoteWorkspaceEvents(mux) return true } + private negotiatedPtyConsumerOwner(serverBuildId?: string): SshPtyConsumerOwnerState | null { + const state = this.ptyConsumerSessionState + if (state && state.mode !== 'legacy-fallback') { + return state as SshPtyConsumerOwnerState + } + const recovery = ptyConsumerRecoveryByTarget.get(this.targetId) + return !serverBuildId || recovery?.serverBuildId === serverBuildId + ? (recovery?.owner ?? null) + : null + } + + private async openPtyConsumerSession( + mux: SshChannelMultiplexer, + serverBuildId: string | undefined + ): Promise { + const previousOwner = this.negotiatedPtyConsumerOwner(serverBuildId) + const options = { + clientInstanceId: this.ptyConsumerClientInstanceId, + expectedServerBuildId: serverBuildId, + allowSameBuildLegacyFallback: true, + outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } + } + try { + return await openSshPtyConsumerSession(mux, { + ...options, + ...(previousOwner + ? { + resume: { + ownerGeneration: previousOwner.ownerGeneration, + ownerLease: previousOwner.ownerLease + } + } + : {}) + }) + } catch (error) { + if ( + !previousOwner || + (error as { code?: unknown }).code !== PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR + ) { + throw error + } + const recovery = ptyConsumerRecoveryByTarget.get(this.targetId) + if (recovery) { + delete recovery.owner + recovery.checkpointsByAppPtyId.clear() + for (const [ptyId, migration] of recovery.modelMigrationsByAppPtyId) { + recovery.modelMigrationsByAppPtyId.set( + ptyId, + migration.then(() => + Object.freeze({ + status: 'checkpoint-unavailable' as const, + reason: 'completion-failed' as const + }) + ) + ) + } + } + this.ptyConsumerSessionState = null + return openSshPtyConsumerSession(mux, options) + } + } + + private rememberPtyConsumerRecovery(serverBuildId: string | undefined): void { + const owner = this.negotiatedPtyConsumerOwner() + if (!owner || !serverBuildId) { + return + } + const previous = ptyConsumerRecoveryByTarget.get(this.targetId) + ptyConsumerRecoveryByTarget.set(this.targetId, { + clientInstanceId: this.ptyConsumerClientInstanceId, + detached: false, + serverBuildId, + owner, + checkpointsByAppPtyId: + previous?.checkpointsByAppPtyId ?? new Map(), + modelMigrationsByAppPtyId: + previous?.modelMigrationsByAppPtyId ?? + new Map>() + }) + } + private configureRelayGraceTime( mux: SshChannelMultiplexer, graceTimeSeconds: number | undefined @@ -874,11 +1197,31 @@ export class SshRelaySession { }) } - private teardownProviders(reason: 'shutdown' | 'connection_lost'): void { + private teardownProviders( + reason: 'shutdown' | 'connection_lost', + outputGenerationReason: string = reason + ): void { this.muxDisposeCleanup?.() this.muxDisposeCleanup = null this.muxNotificationCleanup?.() this.muxNotificationCleanup = null + for (const cleanup of this.ptyRecoveryNotificationCleanups) { + cleanup() + } + this.ptyRecoveryNotificationCleanups = [] + if (this.activePtyProviderGeneration !== null) { + const providerGeneration = this.activePtyProviderGeneration + if (reason === 'connection_lost' && this.negotiatedPtyConsumerOwner()?.outputFlowControl) { + this.beginPtyModelMigration(providerGeneration, outputGenerationReason) + } else { + closeSshPtyOutputGeneration(providerGeneration, outputGenerationReason) + } + this.activePtyProviderGeneration = null + } + this.sourceAckPublisherCleanup?.() + this.sourceAckPublisherCleanup = null + this.sourceCancellationPublisherCleanup?.() + this.sourceCancellationPublisherCleanup = null if (this.mux && !this.mux.isDisposed()) { this.mux.dispose(reason) } @@ -907,6 +1250,15 @@ export class SshRelaySession { unregisterSshPtyProvider(this.targetId) unregisterSshFilesystemProvider(this.targetId) unregisterSshGitProvider(this.targetId) + this.sourceIdentityByRelayPtyId.clear() + this.retiredSourceDeliveries.clear() + for (const pending of this.pendingPtyReattaches.values()) { + for (const resolve of pending.recoveryWaiters) { + resolve() + } + } + this.pendingPtyReattaches.clear() + this.ptyRecoveryRetention.clear() } // Why: back-compat for old relays that gate FS ops on registered roots; removable post-cutover (docs/relay-fs-allowlist-removal.md). @@ -978,68 +1330,344 @@ export class SshRelaySession { } } - private wireUpPtyEvents(ptyProvider: SshPtyProvider): void { + private wireUpPtyEvents( + ptyProvider: SshPtyProvider, + mux: SshChannelMultiplexer, + providerGeneration: number + ): void { ptyProvider.onData((payload) => { - const rawLength = payload.sequenceChars ?? payload.data.length - const seq = this.runtime?.onPtyData( - payload.id, - payload.data, - Date.now(), - rawLength, - payload.transformed - ) - const win = this.getMainWindow() - if (!win || win.isDestroyed()) { + if ( + this.mux !== mux || + this.activePtyProviderGeneration !== providerGeneration || + payload.providerGeneration !== providerGeneration + ) { return } - // Why: hidden-delivery gate parity with ipc/pty.ts — latch model-restore out-of-band, never an in-band pty:data sentinel (OSC-9999-only chunks strip to empty). - const store = this.store as { getSettings?: Store['getSettings'] } - if (shouldDropHiddenRendererPtyData(payload.id, store.getSettings?.())) { - const drop = recordHiddenRendererPtyDataDrop(payload.id, rawLength) - if (drop.shouldEmitRestoreMarker) { - win.webContents.send('pty:modelRestoreNeeded', { - id: payload.id, - reason: 'hidden-drop', - ...(typeof seq === 'number' ? { markerSeq: seq } : {}) - } satisfies PtyModelRestoreNeededEvent) + const pending = this.pendingPtyReattaches.get(payload.id) + if (pending && this.negotiatedPtyConsumerOwner()?.outputFlowControl) { + if (pending.livePassthrough) { + void this.acceptPtyData(payload).catch(() => {}) + return } + this.quarantineReattachData(pending, payload) return } - if (payload.data.length > 0 || payload.transformed) { - win.webContents.send('pty:data', { - ...payload, - ...(typeof seq === 'number' ? { seq } : {}), - rawLength, - ...(payload.transformed ? { transformed: true } : {}) - }) - } + void this.acceptPtyData(payload).catch(() => {}) }) ptyProvider.onReplay((payload) => { + if (this.mux !== mux || this.activePtyProviderGeneration !== providerGeneration) { + return + } const win = this.getMainWindow() if (win && !win.isDestroyed()) { win.webContents.send('pty:replay', payload) } }) ptyProvider.onExit((payload) => { + if ( + this.mux !== mux || + this.activePtyProviderGeneration !== providerGeneration || + payload.providerGeneration !== providerGeneration + ) { + return + } const pendingReattach = this.pendingPtyReattaches.get(payload.id) - if (pendingReattach) { + if (pendingReattach && !pendingReattach.activated) { // Why: attach response and exit can share one transport batch, before incarnation restoration runs. pendingReattach.exits.push(payload) + this.wakeRecovery(pendingReattach) return } if (!isCurrentPtyExit(payload)) { return } - this.retireExitedPty(payload) + void this.acceptPtyExit(payload).catch(() => {}) }) } - private retireExitedPty(payload: SshPtyExitPayload): void { + private acceptPtyData(payload: SshPtyDataPayload): Promise { + const consumerOwner = this.negotiatedPtyConsumerOwner() + const offeredSource = payload.source + if ( + offeredSource && + this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource) + ) { + return Promise.resolve() + } + if ( + consumerOwner?.outputFlowControl && + (!offeredSource || + payload.sourceMalformed || + offeredSource.clientGeneration !== consumerOwner.clientGeneration || + offeredSource.ownerGeneration !== consumerOwner.ownerGeneration) + ) { + closeSshPtyOutputGeneration( + payload.providerGeneration, + 'ssh_source_frame_malformed_or_missing' + ) + this.mux?.dispose('connection_lost') + return Promise.reject(new Error('ssh_source_frame_malformed_or_missing')) + } + const source = consumerOwner?.outputFlowControl ? offeredSource : undefined + if (source) { + const current = this.sourceIdentityByRelayPtyId.get(source.relayPtyId) + if ( + source.sourceEndSu <= source.sourceStartSu || + (current && + (current.deliveryToken !== source.deliveryToken || + current.clientGeneration !== source.clientGeneration || + current.ownerGeneration !== source.ownerGeneration || + current.ptyIncarnation !== payload.ptyIncarnation || + (current.nextSourceSu !== undefined && current.nextSourceSu !== source.sourceStartSu))) + ) { + closeSshPtyOutputGeneration( + payload.providerGeneration, + 'ssh_source_frame_stale_or_non_contiguous' + ) + this.mux?.dispose('connection_lost') + return Promise.reject(new Error('ssh_source_frame_stale_or_non_contiguous')) + } + this.sourceIdentityByRelayPtyId.set(source.relayPtyId, { + deliveryToken: source.deliveryToken, + clientGeneration: source.clientGeneration, + ownerGeneration: source.ownerGeneration, + ptyIncarnation: payload.ptyIncarnation, + nextSourceSu: source.sourceEndSu + }) + } + const rawLength = payload.sequenceChars ?? payload.data.length + return acceptSshPtyOutputData({ + id: payload.id, + data: payload.data, + providerGeneration: payload.providerGeneration, + ptyIncarnation: payload.ptyIncarnation, + rawLength, + transformed: payload.transformed === true, + ...(typeof payload.seq === 'number' ? { sequence: payload.seq } : {}), + ...(source ? { source } : {}) + }) + } + + private quarantineReattachData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void { + this.observePrivateRecoveryFrame(pending, payload) + if (pending.restoreRequired) { + return + } + const sourceSu = payload.source + ? payload.source.sourceEndSu - payload.source.sourceStartSu + : (payload.sequenceChars ?? payload.data.length) + if (!this.ptyRecoveryRetention.tryRetain(pending.retentionKey, payload.data, sourceSu)) { + pending.restoreRequired = 'recoveryQuarantineCapacityExceeded' + this.wakeRecovery(pending) + return + } + this.routeQuarantinedReattachData(pending, payload) + } + + private routeQuarantinedReattachData( + pending: PendingPtyReattach, + payload: SshPtyDataPayload + ): void { + this.observePrivateRecoveryFrame(pending, payload) + if (!pending.recovery) { + pending.queuedData.push(payload) + return + } + if ( + pending.recoveryComplete && + pending.nextRecoverySourceSu === pending.recovery.recoveryEndSu + ) { + pending.liveData.push(payload) + return + } + this.admitRecoveryData(pending, payload) + } + + private observePrivateRecoveryFrame( + pending: PendingPtyReattach, + payload: SshPtyDataPayload + ): void { + const recovery = pending.recovery + if ( + recovery && + payload.source?.deliveryToken === recovery.deliveryToken && + payload.source.clientGeneration === recovery.clientGeneration && + payload.source.ownerGeneration === recovery.ownerGeneration && + payload.ptyIncarnation === recovery.ptyIncarnation + ) { + pending.highestRecoverySourceEndSu = Math.max( + pending.highestRecoverySourceEndSu ?? recovery.checkpointSourceEndSu, + payload.source.sourceEndSu + ) + } + } + + private admitRecoveryData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void { + if (pending.restoreRequired) { + return + } + const recovery = pending.recovery + const nextSourceSu = pending.nextRecoverySourceSu + if ( + !recovery || + !payload.source || + nextSourceSu === undefined || + payload.source.deliveryToken !== recovery.deliveryToken || + payload.source.clientGeneration !== recovery.clientGeneration || + payload.source.ownerGeneration !== recovery.ownerGeneration || + payload.source.sourceStartSu !== nextSourceSu || + payload.source.sourceEndSu <= payload.source.sourceStartSu || + payload.source.sourceEndSu > recovery.recoveryEndSu || + payload.ptyIncarnation !== recovery.ptyIncarnation + ) { + pending.restoreRequired = 'recoveryFrameIdentityMismatch' + this.wakeRecovery(pending) + return + } + pending.nextRecoverySourceSu = payload.source.sourceEndSu + pending.recoveryData.push(payload) + } + + private installPtyRecoveryNotifications(mux: SshChannelMultiplexer): void { + for (const cleanup of this.ptyRecoveryNotificationCleanups) { + cleanup() + } + this.ptyRecoveryNotificationCleanups = [ + mux.onNotificationByMethod('pty.recoveryComplete', (params) => { + if (this.mux !== mux) { + return + } + const id = typeof params.id === 'string' ? toAppSshPtyId(this.targetId, params.id) : '' + const pending = this.pendingPtyReattaches.get(id) + if (!pending || pending.mux !== mux) { + return + } + const complete = parseRecoveryComplete(params) + if (!complete) { + pending.restoreRequired = 'invalidRecoveryComplete' + } else { + pending.recoveryComplete = complete + } + this.wakeRecovery(pending) + }), + mux.onNotificationByMethod('pty.restoreRequired', (params) => { + if (this.mux !== mux) { + return + } + const id = typeof params.id === 'string' ? toAppSshPtyId(this.targetId, params.id) : '' + const pending = this.pendingPtyReattaches.get(id) + if (!pending || pending.mux !== mux) { + return + } + pending.restoreRequired = + typeof params.reason === 'string' ? params.reason : 'relayRestoreRequired' + this.wakeRecovery(pending) + }), + mux.onNotificationByMethod('pty.deliveryCanceled', (params) => { + if (this.mux !== mux) { + return + } + const id = typeof params.id === 'string' ? params.id : '' + const identity = this.sourceIdentityByRelayPtyId.get(id) + if ( + !identity || + params.deliveryToken !== identity.deliveryToken || + params.clientGeneration !== identity.clientGeneration || + params.ownerGeneration !== identity.ownerGeneration || + params.ptyIncarnation !== identity.ptyIncarnation + ) { + return + } + const replacementDeliveryToken = + typeof params.replacementDeliveryToken === 'string' ? params.replacementDeliveryToken : '' + const pending = this.pendingPtyReattaches.get(toAppSshPtyId(this.targetId, id)) + if (pending?.mux === mux) { + if ( + replacementDeliveryToken.length === 0 || + replacementDeliveryToken === identity.deliveryToken + ) { + pending.restoreRequired = + typeof params.reason === 'string' + ? `relayDeliveryCanceled:${params.reason}` + : 'relayDeliveryCanceled' + this.wakeRecovery(pending) + return + } + if ( + pending.replacementDeliveryToken && + pending.replacementDeliveryToken !== replacementDeliveryToken + ) { + pending.restoreRequired = 'recoveryReplacementTokenMismatch' + this.wakeRecovery(pending) + return + } + pending.replacementDeliveryToken = replacementDeliveryToken + return + } + const generation = this.activePtyProviderGeneration + if ( + generation !== null && + Number.isSafeInteger(params.sentEndSu) && + Number.isSafeInteger(params.creditedEndSu) + ) { + try { + applySshPtySourceCancellationProof( + { + id: toAppSshPtyId(this.targetId, id), + code: -1, + providerGeneration: generation, + ptyIncarnation: identity.ptyIncarnation + }, + { + sentEndSu: params.sentEndSu as number, + creditedEndSu: params.creditedEndSu as number + } + ) + this.retiredSourceDeliveries.retire(generation, { + relayPtyId: id, + ...identity + }) + this.sourceIdentityByRelayPtyId.delete(id) + } catch { + /* Invalid proof retains the active token identity. */ + } + } + }) + ] + } + + private wakeRecovery(pending: PendingPtyReattach): void { + for (const resolve of pending.recoveryWaiters) { + resolve() + } + pending.recoveryWaiters.clear() + } + + private async acceptPtyExit(payload: SshPtyExitPayload): Promise { + await acceptSshPtyOutputExit({ + id: payload.id, + code: payload.code, + providerGeneration: payload.providerGeneration, + ptyIncarnation: payload.ptyIncarnation + }) + if (isCurrentPtyExit(payload)) { + this.retireExitedPty(payload, true) + } + } + + private retireExitedPty(payload: SshPtyExitPayload, deliveryHandled = false): void { const relayPtyId = toRelaySshPtyId(this.targetId, payload.id) + this.retiredSourceDeliveries.activate(relayPtyId) clearProviderPtyState(payload.id) deletePtyOwnership(payload.id) - this.forwardedReattachReplayByPty.delete(payload.id) + ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.delete(payload.id) + ptyConsumerRecoveryByTarget + .get(this.targetId) + ?.checkpointsByAppPtyId.delete(toRelaySshPtyId(this.targetId, payload.id)) this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'terminated') + if (deliveryHandled) { + return + } this.runtime?.onPtyExit(payload.id, payload.code, payload.incarnationId) const win = this.getMainWindow() if (win && !win.isDestroyed()) { @@ -1047,26 +1675,8 @@ export class SshRelaySession { } } - private replayFingerprint(data: string): string { - const head = data.slice(0, REPLAY_FINGERPRINT_EDGE_CHARS) - const tail = data.slice(-REPLAY_FINGERPRINT_EDGE_CHARS) - return `${data.length}:${head}:${tail}` - } - - private shouldForwardReattachReplay(appPtyId: string, data: string): boolean { - const now = Date.now() - const fingerprint = this.replayFingerprint(data) - const previous = this.forwardedReattachReplayByPty.get(appPtyId) - this.forwardedReattachReplayByPty.set(appPtyId, { fingerprint, deliveredAt: now }) - return ( - !previous || - previous.fingerprint !== fingerprint || - now - previous.deliveredAt > RECONNECT_REPLAY_DUPLICATE_WINDOW_MS - ) - } - private forwardReattachReplay(appPtyId: string, data: string): void { - if (!data || !this.shouldForwardReattachReplay(appPtyId, data)) { + if (!data) { return } const win = this.getMainWindow() @@ -1075,7 +1685,10 @@ export class SshRelaySession { } } - private async reattachKnownPtys(shouldContinue: () => boolean): Promise { + private async reattachKnownPtys( + mux: SshChannelMultiplexer, + shouldContinue: () => boolean + ): Promise { const activeLeases = this.store .getSshRemotePtyLeases(this.targetId) .filter((lease) => lease.state !== 'terminated' && lease.state !== 'expired') @@ -1100,101 +1713,722 @@ export class SshRelaySession { ]) ) const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined - if (!ptyProvider) { + const providerGeneration = this.activePtyProviderGeneration + if (!ptyProvider || providerGeneration === null || this.mux !== mux) { return } - for (const ptyId of ptyIds) { + let nextPtyIndex = 0 + const worker = async (): Promise => { + while (shouldContinue()) { + const ptyId = ptyIds[nextPtyIndex++] + if (ptyId === undefined) { + return + } + try { + await this.reattachKnownPty({ + ptyProvider, + ptyId, + activeLeaseByPtyId, + expectedIdentityByPtyId, + mux, + providerGeneration, + shouldContinue + }) + } catch (error) { + if (isSourceRecoveryCancellationError(error)) { + throw error + } + console.warn( + `[ssh-relay-session] PTY ${ptyId} reattach processing failed for ${this.targetId}: ${ + error instanceof Error ? error.message : String(error) + }` + ) + } + } + } + await Promise.all( + Array.from({ length: Math.min(SSH_PTY_REATTACH_MAX_CONCURRENCY, ptyIds.length) }, worker) + ) + } + + private async reattachKnownPty(args: { + ptyProvider: SshPtyProvider + ptyId: string + activeLeaseByPtyId: Map + expectedIdentityByPtyId: Map + mux: SshChannelMultiplexer + providerGeneration: number + shouldContinue: () => boolean + }): Promise { + const { + ptyProvider, + ptyId, + activeLeaseByPtyId, + expectedIdentityByPtyId, + mux, + providerGeneration, + shouldContinue + } = args + const appPtyId = toAppSshPtyId(this.targetId, ptyId) + const pendingReattach: PendingPtyReattach = { + mux, + providerGeneration, + retentionKey: `${providerGeneration}\0${appPtyId}\0${randomUUID()}`, + exits: [], + queuedData: [], + recoveryData: [], + liveData: [], + recoveryWaiters: new Set(), + livePassthrough: false, + activated: false + } + this.pendingPtyReattaches.set(appPtyId, pendingReattach) + let sourceActivationLease: SshPtyAttachResult['sourceActivationLease'] + let recoveryActivationLease: SshPtyRecoveryActivationLease | undefined + try { + const recoveryRequest = await this.sourceRecoveryRequest(appPtyId) + const attachResult = await this.attachPtyWithRetry( + ptyProvider, + ptyId, + expectedIdentityByPtyId.get(ptyId), + recoveryRequest, + shouldContinue + ) + sourceActivationLease = attachResult.sourceActivationLease if (!shouldContinue()) { return } - const appPtyId = toAppSshPtyId(this.targetId, ptyId) - const pendingReattach: PendingPtyReattach = { exits: [] } - this.pendingPtyReattaches.set(appPtyId, pendingReattach) - try { - const expectedIdentity = expectedIdentityByPtyId.get(ptyId) - const attachResult = - (expectedIdentity - ? await ptyProvider.attachForReconnect(ptyId, expectedIdentity) - : await ptyProvider.attachForReconnect(ptyId)) ?? {} - if (!shouldContinue()) { - return - } - const exitDuringAttach = pendingReattach.exits.find( - (exit) => - !exit.incarnationId || - !attachResult.incarnationId || - exit.incarnationId === attachResult.incarnationId - ) - if (exitDuringAttach) { - if (attachResult.incarnationId) { - restorePtyIncarnation(appPtyId, attachResult.incarnationId) - this.runtime?.acceptPtyIncarnationForExit(appPtyId, attachResult.incarnationId) - } - this.retireExitedPty(exitDuringAttach) - continue - } - setPtyOwnership(appPtyId, this.targetId) + const exitDuringAttach = pendingReattach.exits.find( + (exit) => + !exit.incarnationId || + !attachResult.incarnationId || + exit.incarnationId === attachResult.incarnationId + ) + if (exitDuringAttach && !recoveryRequest) { if (attachResult.incarnationId) { restorePtyIncarnation(appPtyId, attachResult.incarnationId) - const lease = activeLeaseByPtyId.get(ptyId) - if (lease?.worktreeId && lease.tabId && lease.leafId) { - this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { - tabId: lease.tabId, - leafId: lease.leafId, - incarnationId: attachResult.incarnationId - }) - // Why: reconnect may be the first new-relay response that can backfill exact exit fencing. - try { - this.store.persistPtyBinding({ - worktreeId: lease.worktreeId, - tabId: lease.tabId, - leafId: lease.leafId, - ptyId: appPtyId, - incarnationId: attachResult.incarnationId - }) - } catch (error) { - // Why: this backfill improves future fencing but must not disconnect an already-live relay PTY. - console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error) + this.runtime?.acceptPtyIncarnationForExit(appPtyId, attachResult.incarnationId) + } + await this.acceptPtyExit(exitDuringAttach) + return + } + if (recoveryRequest) { + const recovered = await this.finishSourceRecovery( + ptyId, + appPtyId, + attachResult, + recoveryRequest, + pendingReattach, + shouldContinue, + () => { + const lease = sourceActivationLease + if (!lease) { + return } - } else { - this.runtime?.onPtySpawned(appPtyId, attachResult.incarnationId, { - awaitsRegistration: false - }) + recoveryActivationLease = lease.transferToRecovery((payload) => + this.quarantineReattachData(pendingReattach, payload) + ) + sourceActivationLease = undefined } - } - this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached') - this.forwardReattachReplay(appPtyId, attachResult.replay ?? '') - } catch (err) { - if (!isSshPtyNotFoundError(err)) { - throw err - } - if (isSshPtyIdentityMismatchError(err)) { - console.warn( - `[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${ - err instanceof Error ? err.message : String(err) - }` - ) - continue - } - console.warn( - `[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${ - err instanceof Error ? err.message : String(err) - }` ) - clearProviderPtyState(appPtyId) - deletePtyOwnership(appPtyId) - this.forwardedReattachReplayByPty.delete(appPtyId) - this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') - // Why: reattach failure means the remote process is gone; tell the renderer to clear the stale pane. - const win = this.getMainWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) + if (!recovered) { + const recoveryExit = this.findExactPendingExit( + pendingReattach, + attachResult.incarnationId + ) + if ( + recoveryExit && + shouldContinue() && + this.ownsPtyRecoveryAttempt(appPtyId, pendingReattach) + ) { + if (recoveryActivationLease) { + recoveryActivationLease.retire() + recoveryActivationLease = undefined + } else if (sourceActivationLease) { + const canceled = await sourceActivationLease.rollback() + sourceActivationLease = undefined + if (!canceled) { + throw sourceRecoveryCancellationError( + new Error('ssh_source_activation_cancellation_unproven') + ) + } + } + this.preparePtyIncarnationForExit(appPtyId, attachResult.incarnationId) + await this.acceptPtyExit(recoveryExit) + } + return } - } finally { - if (this.pendingPtyReattaches.get(appPtyId) === pendingReattach) { - this.pendingPtyReattaches.delete(appPtyId) + const recoveryExit = this.findExactPendingExit(pendingReattach, attachResult.incarnationId) + if (recoveryExit) { + this.preparePtyIncarnationForExit(appPtyId, attachResult.incarnationId) + pendingReattach.activated = true + recoveryActivationLease?.commit() + recoveryActivationLease = undefined + await this.acceptPtyExit(recoveryExit) + return } } + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pendingReattach)) { + return + } + setPtyOwnership(appPtyId, this.targetId) + if (attachResult.incarnationId) { + restorePtyIncarnation(appPtyId, attachResult.incarnationId) + this.restoreReattachedPtyRuntime( + appPtyId, + attachResult.incarnationId, + activeLeaseByPtyId.get(ptyId) + ) + } + this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached') + pendingReattach.activated = true + recoveryActivationLease?.commit() + recoveryActivationLease = undefined + const exitAfterActivation = pendingReattach.exits.find( + (exit) => + !exit.incarnationId || + !attachResult.incarnationId || + exit.incarnationId === attachResult.incarnationId + ) + if (exitAfterActivation) { + await this.acceptPtyExit(exitAfterActivation) + return + } + if (!recoveryRequest) { + this.forwardReattachReplay(appPtyId, attachResult.replay ?? '') + } + sourceActivationLease?.commit() + sourceActivationLease = undefined + } catch (error) { + if (isSourceRecoveryCancellationError(error)) { + throw error + } + if (!shouldContinue()) { + return + } + this.handlePtyReattachFailure(ptyId, appPtyId, pendingReattach, error) + } finally { + recoveryActivationLease?.retire() + sourceActivationLease?.rollback() + if (this.pendingPtyReattaches.get(appPtyId) === pendingReattach) { + this.pendingPtyReattaches.delete(appPtyId) + } + this.ptyRecoveryRetention.release(pendingReattach.retentionKey) + } + } + + private findExactPendingExit( + pending: PendingPtyReattach, + ptyIncarnation: string | undefined + ): SshPtyExitPayload | undefined { + if (!ptyIncarnation) { + return undefined + } + return pending.exits.find( + (exit) => + exit.providerGeneration === pending.providerGeneration && + exit.ptyIncarnation === ptyIncarnation + ) + } + + private preparePtyIncarnationForExit(appPtyId: string, ptyIncarnation: string | undefined): void { + if (!ptyIncarnation) { + return + } + restorePtyIncarnation(appPtyId, ptyIncarnation) + this.runtime?.acceptPtyIncarnationForExit(appPtyId, ptyIncarnation) + } + + private restoreReattachedPtyRuntime( + appPtyId: string, + incarnationId: string, + lease: SshPtyLease | undefined + ): void { + if (lease?.worktreeId && lease.tabId && lease.leafId) { + this.runtime?.registerPty(appPtyId, lease.worktreeId, this.targetId, { + tabId: lease.tabId, + leafId: lease.leafId, + incarnationId + }) + try { + this.store.persistPtyBinding({ + worktreeId: lease.worktreeId, + tabId: lease.tabId, + leafId: lease.leafId, + ptyId: appPtyId, + incarnationId + }) + } catch (error) { + console.error('[ssh-relay-session] Failed to persist reconnect incarnation:', error) + } + return + } + this.runtime?.onPtySpawned(appPtyId, incarnationId, { awaitsRegistration: false }) + } + + private async attachPtyWithRetry( + ptyProvider: SshPtyProvider, + ptyId: string, + expectedIdentity: ExpectedPtyIdentity | undefined, + recoveryRequest: PtySourceRecoveryRequest | undefined, + shouldContinue: () => boolean + ): Promise { + let lastError: unknown + for (let attempt = 0; attempt < 2; attempt++) { + if (!shouldContinue()) { + throw lastError ?? new Error('PTY reattach attempt is no longer current') + } + try { + return await this.attachPtyWithDeadline( + ptyProvider, + ptyId, + expectedIdentity, + recoveryRequest + ) + } catch (error) { + lastError = error + if (!shouldContinue() || isSshPtyNotFoundError(error) || attempt === 1) { + throw error + } + await this.waitForPtyReattachRetry() + } + } + throw lastError + } + + private async attachPtyWithDeadline( + ptyProvider: SshPtyProvider, + ptyId: string, + expectedIdentity: ExpectedPtyIdentity | undefined, + recoveryRequest: PtySourceRecoveryRequest | undefined + ): Promise { + let timer: ReturnType | undefined + let timedOut = false + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + timedOut = true + reject( + new Error(`PTY reattach attempt timed out after ${SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS}ms`) + ) + }, SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS) + timer.unref?.() + }) + try { + const attach = expectedIdentity + ? recoveryRequest + ? ptyProvider.attachForReconnect(ptyId, expectedIdentity, recoveryRequest) + : ptyProvider.attachForReconnect(ptyId, expectedIdentity) + : recoveryRequest + ? ptyProvider.attachForReconnect(ptyId, undefined, recoveryRequest) + : ptyProvider.attachForReconnect(ptyId) + const guardedAttach = attach.then((result) => { + if (timedOut) { + result.sourceActivationLease?.rollback() + } + return result + }) + return (await Promise.race([guardedAttach, timeout])) ?? {} + } finally { + if (timer) { + clearTimeout(timer) + } } } + + private async waitForPtyReattachRetry(): Promise { + const delayMs = + SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS + + Math.floor(Math.random() * (SSH_PTY_REATTACH_RETRY_JITTER_MS + 1)) + await new Promise((resolve) => { + const timer = setTimeout(resolve, delayMs) + timer.unref?.() + }) + } + + private handlePtyReattachFailure( + ptyId: string, + appPtyId: string, + pending: PendingPtyReattach, + error: unknown + ): void { + if (!isSshPtyNotFoundError(error)) { + pending.restoreRequired = 'reattachAttemptsExhausted' + this.wakeRecovery(pending) + console.warn( + `[ssh-relay-session] Leaving PTY ${ptyId} detached for ${this.targetId} after bounded reattach attempts failed: ${ + error instanceof Error ? error.message : String(error) + }` + ) + return + } + if (isSshPtyIdentityMismatchError(error)) { + console.warn( + `[ssh-relay-session] Ignoring stale PTY ${ptyId} for ${this.targetId} after relay identity mismatch: ${ + error instanceof Error ? error.message : String(error) + }` + ) + return + } + console.warn( + `[ssh-relay-session] Dropping stale PTY ${ptyId} for ${this.targetId} after relay reattach failed: ${ + error instanceof Error ? error.message : String(error) + }` + ) + clearProviderPtyState(appPtyId) + deletePtyOwnership(appPtyId) + this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') + const win = this.getMainWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('pty:exit', { id: appPtyId, code: -1 }) + } + } + + private async sourceRecoveryRequest( + appPtyId: string + ): Promise { + if (!this.negotiatedPtyConsumerOwner()?.outputFlowControl) { + return undefined + } + const recovery = ptyConsumerRecoveryByTarget.get(this.targetId) + const migration = recovery?.modelMigrationsByAppPtyId.get(appPtyId) + if (migration) { + const outcome = await migration + if (recovery?.modelMigrationsByAppPtyId.get(appPtyId) === migration) { + recovery.modelMigrationsByAppPtyId.delete(appPtyId) + } + if (outcome.status !== 'settled') { + return Object.freeze({ status: 'checkpointUnavailable' }) + } + } + const checkpoints = recovery?.checkpointsByAppPtyId + const relayPtyId = toRelaySshPtyId(this.targetId, appPtyId) + const checkpoint = checkpoints?.get(appPtyId) ?? checkpoints?.get(relayPtyId) + if (!checkpoint) { + return Object.freeze({ status: 'checkpointUnavailable' }) + } + return Object.freeze({ + status: 'checkpoint', + clientGeneration: checkpoint.clientGeneration, + ownerGeneration: checkpoint.ownerGeneration, + ptyIncarnation: checkpoint.ptyIncarnation, + deliveryToken: checkpoint.deliveryToken, + acceptedSourceEndSu: checkpoint.acceptedSourceEndSu + }) + } + + private beginPtyModelMigration(providerGeneration: number, closeReason: string): void { + const recovery = ptyConsumerRecoveryByTarget.get(this.targetId) + if (!recovery) { + closeSshPtyOutputGeneration(providerGeneration, closeReason) + return + } + for (const checkpoint of getSshPtyAcceptedSourceCheckpoints(providerGeneration)) { + recovery.checkpointsByAppPtyId.set(checkpoint.id, checkpoint) + } + const migration = beginSshPtyOutputGenerationMigration(providerGeneration) + for (const [ptyId, result] of migration.byPty) { + const previous = recovery.modelMigrationsByAppPtyId.get(ptyId) + const fence = previous ? previous.then(() => result) : result + recovery.modelMigrationsByAppPtyId.set(ptyId, fence) + void fence.then((outcome) => { + const current = ptyConsumerRecoveryByTarget.get(this.targetId) + if (current?.modelMigrationsByAppPtyId.get(ptyId) !== fence) { + return + } + if (outcome.status === 'settled') { + current.checkpointsByAppPtyId.set(ptyId, outcome.checkpoint) + } else { + current.checkpointsByAppPtyId.delete(ptyId) + current.checkpointsByAppPtyId.delete(toRelaySshPtyId(this.targetId, ptyId)) + } + current.modelMigrationsByAppPtyId.delete(ptyId) + }) + } + void migration.completion.then(() => { + closeSshPtyOutputGeneration(providerGeneration, closeReason) + }) + } + + private async finishSourceRecovery( + relayPtyId: string, + appPtyId: string, + attachResult: SshPtyAttachResult, + request: PtySourceRecoveryRequest, + pending: PendingPtyReattach, + shouldContinue: () => boolean, + activateRecoveryQuarantine: () => void + ): Promise { + const recovery = attachResult.sourceRecovery + const pendingRecovery = recovery?.status === 'pending' ? recovery : undefined + const owner = this.negotiatedPtyConsumerOwner() + if ( + !owner?.outputFlowControl || + !pendingRecovery || + request.status !== 'checkpoint' || + pendingRecovery.clientGeneration !== owner.clientGeneration || + pendingRecovery.ownerGeneration !== owner.ownerGeneration || + pendingRecovery.ptyIncarnation !== attachResult.incarnationId || + pendingRecovery.ptyIncarnation !== request.ptyIncarnation || + pendingRecovery.checkpointSourceEndSu !== request.acceptedSourceEndSu || + (pending.replacementDeliveryToken !== undefined && + pending.replacementDeliveryToken !== pendingRecovery.deliveryToken) + ) { + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) + return false + } + const acceptedRecovery = pendingRecovery + pending.recovery = acceptedRecovery + pending.nextRecoverySourceSu = acceptedRecovery.checkpointSourceEndSu + this.retiredSourceDeliveries.activate(relayPtyId) + this.sourceIdentityByRelayPtyId.set(relayPtyId, { + deliveryToken: acceptedRecovery.deliveryToken, + clientGeneration: acceptedRecovery.clientGeneration, + ownerGeneration: acceptedRecovery.ownerGeneration, + ptyIncarnation: acceptedRecovery.ptyIncarnation, + nextSourceSu: acceptedRecovery.checkpointSourceEndSu + }) + activateRecoveryQuarantine() + for (const payload of pending.queuedData.splice(0)) { + this.routeQuarantinedReattachData(pending, payload) + } + await this.waitForRecoveryFence(pending, shouldContinue) + const exactExit = this.findExactPendingExit(pending, acceptedRecovery.ptyIncarnation) + const complete = pending.recoveryComplete ?? (exactExit ? acceptedRecovery : undefined) + if ( + !shouldContinue() || + pending.restoreRequired || + !complete || + complete.deliveryToken !== acceptedRecovery.deliveryToken || + complete.clientGeneration !== acceptedRecovery.clientGeneration || + complete.ownerGeneration !== acceptedRecovery.ownerGeneration || + complete.ptyIncarnation !== acceptedRecovery.ptyIncarnation || + complete.checkpointSourceEndSu !== acceptedRecovery.checkpointSourceEndSu || + complete.recoveryEndSu !== acceptedRecovery.recoveryEndSu || + pending.nextRecoverySourceSu !== acceptedRecovery.recoveryEndSu + ) { + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) + return false + } + let nextLiveSourceSu = acceptedRecovery.recoveryEndSu + for (const payload of pending.liveData) { + if ( + !payload.source || + payload.source.deliveryToken !== acceptedRecovery.deliveryToken || + payload.source.clientGeneration !== acceptedRecovery.clientGeneration || + payload.source.ownerGeneration !== acceptedRecovery.ownerGeneration || + payload.source.sourceStartSu !== nextLiveSourceSu || + payload.source.sourceEndSu <= payload.source.sourceStartSu || + payload.ptyIncarnation !== acceptedRecovery.ptyIncarnation + ) { + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) + return false + } + nextLiveSourceSu = payload.source.sourceEndSu + } + try { + for (const payload of pending.recoveryData) { + await this.acceptPtyData(payload) + } + for (const payload of pending.liveData) { + await this.acceptPtyData(payload) + } + pending.livePassthrough = true + } catch { + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending) + return false + } + if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return false + } + const acceptedSourceEndSu = pending.liveData.reduce( + (endSu, payload) => Math.max(endSu, payload.source?.sourceEndSu ?? endSu), + acceptedRecovery.recoveryEndSu + ) + ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.set( + relayPtyId, + Object.freeze({ + id: relayPtyId, + providerGeneration: this.activePtyProviderGeneration!, + clientGeneration: acceptedRecovery.clientGeneration, + ownerGeneration: acceptedRecovery.ownerGeneration, + ptyIncarnation: acceptedRecovery.ptyIncarnation, + deliveryToken: acceptedRecovery.deliveryToken, + acceptedSourceEndSu + }) + ) + return true + } + + private async waitForRecoveryFence( + pending: PendingPtyReattach, + shouldContinue: () => boolean + ): Promise { + const deadline = Date.now() + SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS + while ( + shouldContinue() && + !pending.recoveryComplete && + !pending.restoreRequired && + !this.findExactPendingExit(pending, pending.recovery?.ptyIncarnation) && + Date.now() < deadline + ) { + await new Promise((resolve) => { + const timer = setTimeout( + () => { + pending.recoveryWaiters.delete(settle) + resolve() + }, + Math.max(1, deadline - Date.now()) + ) + timer.unref?.() + const settle = (): void => { + clearTimeout(timer) + resolve() + } + pending.recoveryWaiters.add(settle) + }) + } + if ( + !pending.recoveryComplete && + !pending.restoreRequired && + !this.findExactPendingExit(pending, pending.recovery?.ptyIncarnation) + ) { + pending.restoreRequired = 'recoveryFenceTimeout' + } + } + + private async abandonPtySourceRecovery( + relayPtyId: string, + appPtyId: string, + pending: PendingPtyReattach + ): Promise { + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + const recovery = pending.recovery + const { mux, providerGeneration } = pending + if (recovery && !mux.isDisposed()) { + const cancellationRequest = { + id: relayPtyId, + clientGeneration: recovery.clientGeneration, + ownerGeneration: recovery.ownerGeneration, + deliveryToken: recovery.deliveryToken + } + this.retiredSourceDeliveries.retire(providerGeneration, { + relayPtyId, + deliveryToken: recovery.deliveryToken, + clientGeneration: recovery.clientGeneration, + ownerGeneration: recovery.ownerGeneration + }) + try { + const result = (await mux.request('pty.cancelDelivery', cancellationRequest)) as Record< + string, + unknown + > + const highestPrivateSourceEndSu = pending.liveData.reduce( + (endSu, payload) => Math.max(endSu, payload.source?.sourceEndSu ?? endSu), + Math.max( + pending.nextRecoverySourceSu ?? recovery.checkpointSourceEndSu, + pending.highestRecoverySourceEndSu ?? recovery.checkpointSourceEndSu + ) + ) + if ( + result.canceled !== true || + !Number.isSafeInteger(result.sentEndSu) || + (result.sentEndSu as number) < highestPrivateSourceEndSu || + !Number.isSafeInteger(result.creditedEndSu) || + result.creditedEndSu !== recovery.checkpointSourceEndSu || + (result.creditedEndSu as number) > (result.sentEndSu as number) + ) { + throw new Error('ssh_source_cancellation_proof_invalid') + } + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + const identity = this.sourceIdentityByRelayPtyId.get(relayPtyId) + if (identity && !this.sameSourceDelivery(identity, recovery)) { + return + } + if (identity) { + const applied = applySshPtySourceRecoveryCancellationProof( + { + id: appPtyId, + code: -1, + providerGeneration, + ptyIncarnation: recovery.ptyIncarnation + }, + { + sentEndSu: result.sentEndSu as number, + creditedEndSu: result.creditedEndSu as number + } + ) + if (!applied) { + throw new Error('ssh_source_cancellation_proof_rejected') + } + } + } catch (error) { + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + console.warn( + `[ssh-relay-session] Failed to cancel replacement delivery for ${relayPtyId}: ${ + error instanceof Error ? error.message : String(error) + }` + ) + throw sourceRecoveryCancellationError(error) + } + } + if (!this.ownsPtyRecoveryAttempt(appPtyId, pending)) { + return + } + const identity = this.sourceIdentityByRelayPtyId.get(relayPtyId) + if (!identity || !recovery || this.sameSourceDelivery(identity, recovery)) { + this.sourceIdentityByRelayPtyId.delete(relayPtyId) + } + ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.delete(appPtyId) + ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.delete(relayPtyId) + this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'detached') + } + + private ownsPtyRecoveryAttempt(appPtyId: string, pending: PendingPtyReattach): boolean { + return ( + this.pendingPtyReattaches.get(appPtyId) === pending && + this.mux === pending.mux && + this.activePtyProviderGeneration === pending.providerGeneration && + !pending.mux.isDisposed() + ) + } + + private sameSourceDelivery( + left: Readonly<{ + deliveryToken: string + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + }>, + right: Readonly<{ + deliveryToken: string + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + }> + ): boolean { + return ( + left.deliveryToken === right.deliveryToken && + left.clientGeneration === right.clientGeneration && + left.ownerGeneration === right.ownerGeneration && + left.ptyIncarnation === right.ptyIncarnation + ) + } } diff --git a/src/main/ssh/ssh-relay-sftp-namespace-install.test.ts b/src/main/ssh/ssh-relay-sftp-namespace-install.test.ts index a9e33e62ca1..324781486de 100644 --- a/src/main/ssh/ssh-relay-sftp-namespace-install.test.ts +++ b/src/main/ssh/ssh-relay-sftp-namespace-install.test.ts @@ -238,6 +238,7 @@ const POSIX_FIRST_INSTALL = [ 'ORCA-NPTY-PROBE-OK\n', '', // rm probe stderr 'DEAD', + '', // publish the per-launch credential 'READY' ] @@ -253,6 +254,17 @@ const POSIX_REPAIR = [ 'ORCA-NPTY-PROBE-OK\n', '', // rm probe stderr 'DEAD', + '', // publish the per-launch credential + 'READY' +] + +const POSIX_HEALTHY_RECONNECT = [ + '__ORCA_REMOTE_PLATFORM__ Linux x86_64', + SHELL_HOME, + 'ORCA-NATIVE-DEPS-OK', + '', // per-launch namespace marker + 'DEAD', + '', // publish the per-launch credential 'READY' ] @@ -280,7 +292,7 @@ describe('relay install writes on a split SFTP namespace', () => { vi.restoreAllMocks() }) - it('redirects every first-install write while shell commands keep the canonical path', async () => { + it('redirects every first-install artifact transfer while shell commands stay canonical', async () => { const conn = makeConnection(capture) feed(POSIX_FIRST_INSTALL) @@ -310,7 +322,7 @@ describe('relay install writes on a split SFTP namespace', () => { expect(markerCommands[0]).toContain(`${SHELL_RELAY_DIR}/.install-lock`) }) - it('probes one shared marker for every write of an install', async () => { + it('probes one shared marker for every first-install artifact transfer', async () => { const conn = makeConnection(capture) feed(POSIX_FIRST_INSTALL) @@ -514,6 +526,88 @@ describe('relay repair writes on a split SFTP namespace', () => { vi.restoreAllMocks() }) + it('publishes a healthy reconnect credential in the canonical shell namespace', async () => { + const conn = makeConnection(capture, { transferMethods: true }) + feed(POSIX_HEALTHY_RECONNECT) + + await deployAndLaunchRelay(conn) + + expect(capture.writePaths).toEqual([]) + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) + }) + + it('does not redirect a healthy reconnect credential through fallback SFTP', async () => { + const conn = makeConnection(capture) + feed(POSIX_HEALTHY_RECONNECT) + + await deployAndLaunchRelay(conn) + + expect(capture.writePaths).toEqual([]) + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) + }) + + it.each(['busy', 'error'] as const)( + 'generates a healthy %s-lock credential in the canonical shell namespace', + async (lockResult) => { + const conn = makeConnection(capture) + vi.mocked(tryAcquireRelayRepairLock).mockResolvedValue(lockResult) + feed([ + '__ORCA_REMOTE_PLATFORM__ Linux x86_64', + SHELL_HOME, + 'ORCA-NATIVE-DEPS-OK', + 'DEAD', + '', // remote credential generation + 'READY' + ]) + + await deployAndLaunchRelay(conn) + + expect(capture.writePaths).toEqual([]) + expect(execCommands().some((command) => MARKER_PATTERN.test(command))).toBe(false) + const credentialCommand = execCommands().find((command) => command.includes('randomBytes')) + expect(credentialCommand).toContain(`${SHELL_RELAY_DIR}/relay.sock.credential`) + expect(credentialCommand).not.toContain(SFTP_RELAY_DIR) + } + ) + + it('keeps a healthy system-SSH reconnect on its secure shell-path writer', async () => { + const conn = makeConnection(capture, { systemSsh: true, transferMethods: true }) + feed([ + '__ORCA_REMOTE_PLATFORM__ Linux x86_64', + SHELL_HOME, + 'ORCA-NATIVE-DEPS-OK', + 'DEAD', + '', // remote credential generation + 'READY' + ]) + + await deployAndLaunchRelay(conn) + + expect(conn.sftp).not.toHaveBeenCalled() + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) + expect(capture.writePaths).toEqual([]) + }) + + it('falls back to remote credential generation when a healthy marker is unavailable', async () => { + const conn = makeConnection(capture) + feed(['__ORCA_REMOTE_PLATFORM__ Linux x86_64', SHELL_HOME, 'ORCA-NATIVE-DEPS-OK']) + vi.mocked(execCommand).mockRejectedValueOnce(new Error('read-only marker')) + feed(['DEAD', '', 'READY']) + + await deployAndLaunchRelay(conn) + + expect(capture.writePaths).toEqual([]) + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) + }) + it('stamps the marker only after the locked recheck, then redirects package.json', async () => { const conn = makeConnection(capture) let execCountAtLock = -1 @@ -545,6 +639,7 @@ describe('relay repair writes on a split SFTP namespace', () => { 'ORCA-NPTY-PROBE-OK\n', '', // rm probe stderr 'DEAD', + '', // remote credential generation 'READY' ]) @@ -554,6 +649,9 @@ describe('relay repair writes on a split SFTP namespace', () => { expect(conn.sftp).not.toHaveBeenCalled() expect(capture.writePaths).toEqual([`${SHELL_RELAY_DIR}/package.json`]) expect(capture.writeOptions).toEqual([expect.objectContaining({ sftpNamespace: undefined })]) + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) }) it('degrades to shell paths when marker creation fails outright', async () => { @@ -566,12 +664,16 @@ describe('relay repair writes on a split SFTP namespace', () => { 'ORCA-NPTY-PROBE-OK\n', '', // rm probe stderr 'DEAD', + '', // remote credential generation 'READY' ]) await deployAndLaunchRelay(conn) expect(capture.writePaths).toEqual([`${SHELL_RELAY_DIR}/package.json`]) + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) expect(capture.realpathCalls).toEqual([]) expect(warnSpy.mock.calls.map((args) => String(args[0]))).toContainEqual( expect.stringContaining('SFTP namespace marker unavailable') @@ -584,12 +686,14 @@ describe('relay repair writes on a split SFTP namespace', () => { vi.mocked(execCommand).mockRejectedValueOnce( Object.assign(new Error('marker teardown unconfirmed'), { sshChannelCloseConfirmed: false }) ) - feed(['DEAD', 'READY']) + feed(['DEAD', '', 'READY']) await deployAndLaunchRelay(conn) - // Repair is best-effort: the relay still launches, but nothing was written and no lock was released. expect(capture.writePaths).toEqual([]) + expect(execCommands().find((command) => command.includes('randomBytes'))).toContain( + `${SHELL_RELAY_DIR}/relay.sock.credential` + ) expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled() expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled() expect(warnSpy.mock.calls.map((args) => String(args[0]))).toContainEqual( diff --git a/src/main/ssh/ssh-remote-cli-launcher.test.ts b/src/main/ssh/ssh-remote-cli-launcher.test.ts index 62aa7bec660..3abdd0fcb73 100644 --- a/src/main/ssh/ssh-remote-cli-launcher.test.ts +++ b/src/main/ssh/ssh-remote-cli-launcher.test.ts @@ -29,6 +29,7 @@ describe('SSH remote Orca CLI launcher', () => { relayDir: 'C:/Users/me user/.orca-remote/relay-v1', nodePath: 'C:/Program Files/nodejs/node.exe', sockPath: '\\\\.\\pipe\\orca-relay-123', + credentialFile: 'C:/Users/me user/.orca-remote/relay-v1/relay.sock.credential', hostPlatform: getRemoteHostPlatform('win32-x64') }) } @@ -41,6 +42,7 @@ describe('SSH remote Orca CLI launcher', () => { expect(plan.files[0]?.path).toBe('C:/Users/me user/.orca-relay/bin/orca-launcher.cs') expect(plan.files[0]?.contents).toContain('ProcessStartInfo') expect(plan.files[0]?.contents).toContain('"--orca-cli"') + expect(plan.files[0]?.contents).toContain('socketPath + ".credential"') expect(plan.files[0]?.contents).toContain("value[index] == '\"'") expect(plan.files[0]?.contents).toContain("character == '\\\\'") expect(plan.files[0]?.contents).not.toContain('cmd.exe') @@ -83,11 +85,13 @@ describe('SSH remote Orca CLI launcher', () => { const binDir = join(root, 'bin').replaceAll('\\', '/') const relayDir = join(root, 'relay').replaceAll('\\', '/') const sockPath = '\\\\.\\pipe\\orca-relay-test' + const credentialFile = `${relayDir}/relay.sock.credential` const plan = createRemoteCliInstallPlan({ binDir, relayDir, nodePath: process.execPath, sockPath, + credentialFile, hostPlatform: getRemoteHostPlatform('win32-x64') }) for (const file of plan.files) { @@ -127,7 +131,8 @@ describe('SSH remote Orca CLI launcher', () => { ...process.env, ORCA_RELAY_NODE_PATH: process.execPath, ORCA_RELAY_DIR: relayDir, - ORCA_RELAY_SOCKET_PATH: sockPath + ORCA_RELAY_SOCKET_PATH: sockPath, + ORCA_RELAY_CREDENTIAL_FILE: credentialFile } } ) @@ -136,6 +141,8 @@ describe('SSH remote Orca CLI launcher', () => { expect(JSON.parse(launched.stdout)).toEqual([ '--sock-path', sockPath, + '--credential-file', + credentialFile, '--orca-cli', 'orchestration', 'send', @@ -143,6 +150,26 @@ describe('SSH remote Orca CLI launcher', () => { body, '--json' ]) + + const defaulted = spawnSync(plan.launcherPath, ['status'], { + encoding: 'utf8', + env: { + ...process.env, + ORCA_RELAY_NODE_PATH: process.execPath, + ORCA_RELAY_DIR: relayDir, + ORCA_RELAY_SOCKET_PATH: sockPath, + ORCA_RELAY_CREDENTIAL_FILE: '' + } + }) + expect(defaulted.status, defaulted.stderr).toBe(0) + expect(JSON.parse(defaulted.stdout)).toEqual([ + '--sock-path', + sockPath, + '--credential-file', + `${sockPath}.credential`, + '--orca-cli', + 'status' + ]) } finally { rmSync(root, { recursive: true, force: true }) } @@ -161,6 +188,7 @@ describe('SSH remote Orca CLI launcher', () => { relayDir: join(root, 'relay').replaceAll('\\', '/'), nodePath: process.execPath, sockPath: '\\\\.\\pipe\\orca-relay-test', + credentialFile: join(root, 'relay', 'relay.sock.credential').replaceAll('\\', '/'), hostPlatform: getRemoteHostPlatform('win32-x64') }) for (const file of plan.files) { diff --git a/src/main/ssh/ssh-remote-cli-launcher.ts b/src/main/ssh/ssh-remote-cli-launcher.ts index 4122e0dc09d..aaa3d09b6ce 100644 --- a/src/main/ssh/ssh-remote-cli-launcher.ts +++ b/src/main/ssh/ssh-remote-cli-launcher.ts @@ -7,6 +7,7 @@ type RemoteCliInstallEnv = { relayDir: string nodePath: string sockPath: string + credentialFile?: string hostPlatform: RemoteHostPlatform } @@ -35,6 +36,11 @@ internal static class OrcaRemoteCliLauncher string nodePath = RequireEnvironmentVariable("ORCA_RELAY_NODE_PATH"); string relayDirectory = RequireEnvironmentVariable("ORCA_RELAY_DIR"); string socketPath = RequireEnvironmentVariable("ORCA_RELAY_SOCKET_PATH"); + string credentialFile = Environment.GetEnvironmentVariable("ORCA_RELAY_CREDENTIAL_FILE"); + if (String.IsNullOrEmpty(credentialFile)) + { + credentialFile = socketPath + ".credential"; + } string relayPath = Path.Combine(relayDirectory, "relay.js"); if (!File.Exists(nodePath)) @@ -51,7 +57,7 @@ internal static class OrcaRemoteCliLauncher ProcessStartInfo startInfo = new ProcessStartInfo { FileName = nodePath, - Arguments = BuildArguments(relayPath, socketPath, args), + Arguments = BuildArguments(relayPath, socketPath, credentialFile, args), UseShellExecute = false }; @@ -78,12 +84,14 @@ internal static class OrcaRemoteCliLauncher return value; } - private static string BuildArguments(string relayPath, string socketPath, string[] args) + private static string BuildArguments(string relayPath, string socketPath, string credentialFile, string[] args) { StringBuilder commandLine = new StringBuilder(); AppendArgument(commandLine, relayPath); AppendArgument(commandLine, "--sock-path"); AppendArgument(commandLine, socketPath); + AppendArgument(commandLine, "--credential-file"); + AppendArgument(commandLine, credentialFile); AppendArgument(commandLine, "--orca-cli"); foreach (string arg in args) { @@ -223,11 +231,12 @@ export function createRemoteCliInstallPlan(env: RemoteCliInstallEnv): RemoteCliI `ORCA_RELAY_NODE_PATH=\${ORCA_RELAY_NODE_PATH:-${quoteSh(env.nodePath)}}`, `ORCA_RELAY_DIR=\${ORCA_RELAY_DIR:-${quoteSh(env.relayDir)}}`, `ORCA_RELAY_SOCKET_PATH=\${ORCA_RELAY_SOCKET_PATH:-${quoteSh(env.sockPath)}}`, + `ORCA_RELAY_CREDENTIAL_FILE=\${ORCA_RELAY_CREDENTIAL_FILE:-${quoteSh(env.credentialFile ?? `${env.sockPath}.credential`)}}`, 'if [ ! -S "$ORCA_RELAY_SOCKET_PATH" ]; then', ' echo "Orca SSH CLI bridge cannot find the relay socket: $ORCA_RELAY_SOCKET_PATH" >&2', ' exit 1', 'fi', - 'exec "$ORCA_RELAY_NODE_PATH" "$ORCA_RELAY_DIR/relay.js" --sock-path "$ORCA_RELAY_SOCKET_PATH" --orca-cli "$@"', + 'exec "$ORCA_RELAY_NODE_PATH" "$ORCA_RELAY_DIR/relay.js" --sock-path "$ORCA_RELAY_SOCKET_PATH" --credential-file "$ORCA_RELAY_CREDENTIAL_FILE" --orca-cli "$@"', '' ].join('\n') } diff --git a/src/relay/dispatcher-client-writer.test.ts b/src/relay/dispatcher-client-writer.test.ts new file mode 100644 index 00000000000..13e49b59e8e --- /dev/null +++ b/src/relay/dispatcher-client-writer.test.ts @@ -0,0 +1,349 @@ +import { describe, expect, it, vi } from 'vitest' +import { + DispatcherClientWriter, + type DispatcherWriterLane, + type SinkWriteSettlement +} from './dispatcher-client-writer' + +type AcceptedWrite = { + data: string + settle: (result: SinkWriteSettlement) => void +} + +class FakeSink { + readonly accepted: AcceptedWrite[] = [] + readonly drainWaiters: (() => void)[] = [] + writableLength = 0 + highWaterMark = 4096 + saturateNext = false + closed = false + + write = (data: Buffer, settle: (result: SinkWriteSettlement) => void): boolean | void => { + this.accepted.push({ data: data.toString(), settle }) + this.writableLength += data.length + if (this.saturateNext) { + this.saturateNext = false + return false + } + return true + } + + drain(): void { + this.writableLength = 0 + for (const waiter of this.drainWaiters.splice(0)) { + waiter() + } + } +} + +function createWriter( + sink: FakeSink, + onClosed = vi.fn(), + producerQueueMaxBytes = 1024 +): DispatcherClientWriter { + return new DispatcherClientWriter( + sink.write, + { + supportsWriteCallback: true, + writableLength: () => sink.writableLength, + writableHighWaterMark: () => sink.highWaterMark, + waitWriteDrain: (callback) => { + sink.drainWaiters.push(callback) + return () => { + const index = sink.drainWaiters.indexOf(callback) + if (index >= 0) { + sink.drainWaiters.splice(index, 1) + } + } + }, + close: () => { + sink.closed = true + } + }, + onClosed, + producerQueueMaxBytes + ) +} + +function enqueue( + writer: DispatcherClientWriter, + lane: DispatcherWriterLane, + value: string, + settled = vi.fn() +): ReturnType { + expect(writer.enqueue(lane, () => Buffer.from(value), Buffer.byteLength(value), settled)).toBe( + true + ) + return settled +} + +describe('DispatcherClientWriter', () => { + it('accepts write(false) once and stops ordinary writes until drain', () => { + const sink = new FakeSink() + const writer = createWriter(sink) + sink.saturateNext = true + + const firstSettled = enqueue(writer, 'ordinary', 'first') + enqueue(writer, 'ordinary', 'second') + + expect(sink.accepted.map((write) => write.data)).toEqual(['first']) + sink.accepted[0].settle({ ok: true }) + expect(firstSettled).toHaveBeenCalledWith({ ok: true }) + expect(sink.accepted.map((write) => write.data)).toEqual(['first']) + + sink.drain() + expect(sink.accepted.map((write) => write.data)).toEqual(['first', 'second']) + }) + + it('closes on a callback error without settling queued work twice', () => { + const sink = new FakeSink() + const onClosed = vi.fn() + const writer = createWriter(sink, onClosed) + sink.saturateNext = true + + const firstSettled = enqueue(writer, 'ordinary', 'first') + const secondSettled = enqueue(writer, 'ordinary', 'second') + const error = new Error('async write failed') + sink.accepted[0].settle({ ok: false, error }) + sink.accepted[0].settle({ ok: true }) + + expect(firstSettled).toHaveBeenCalledTimes(1) + expect(firstSettled).toHaveBeenCalledWith({ ok: false, error }) + expect(secondSettled).toHaveBeenCalledTimes(1) + expect(onClosed).toHaveBeenCalledWith(error) + expect(sink.closed).toBe(true) + }) + + it('preserves FIFO within lanes and rechecks control before producer work', () => { + const sink = new FakeSink() + const writer = createWriter(sink) + sink.saturateNext = true + + enqueue(writer, 'ordinary', 'blocker') + enqueue(writer, 'bulk', 'bulk-1') + enqueue(writer, 'ordinary', 'pty-1') + enqueue(writer, 'control', 'control-1') + enqueue(writer, 'ordinary', 'pty-2') + enqueue(writer, 'control', 'control-2') + expect(sink.accepted.map((write) => write.data)).toEqual(['blocker']) + + sink.drain() + + expect(sink.accepted.map((write) => write.data)).toEqual([ + 'blocker', + 'control-1', + 'control-2', + 'pty-1', + 'pty-2', + 'bulk-1' + ]) + }) + + it('allows one coalesced liveness bypass during a saturated epoch', () => { + const sink = new FakeSink() + const writer = createWriter(sink) + sink.saturateNext = true + + enqueue(writer, 'ordinary', 'ordinary') + const first = enqueue(writer, 'liveness', 'keepalive-1') + const replaced = enqueue(writer, 'liveness', 'keepalive-2') + const latest = enqueue(writer, 'liveness', 'keepalive-3') + + expect(sink.accepted.map((write) => write.data)).toEqual(['ordinary', 'keepalive-1']) + expect(first).not.toHaveBeenCalled() + expect(replaced).toHaveBeenCalledOnce() + expect(replaced).toHaveBeenCalledWith({ ok: true }) + expect(latest).not.toHaveBeenCalled() + + sink.drain() + expect(sink.accepted.map((write) => write.data)).toEqual([ + 'ordinary', + 'keepalive-1', + 'keepalive-3' + ]) + }) + + it('gives queued bulk a turn after four interactive or ordinary writes', () => { + const sink = new FakeSink() + const writer = createWriter(sink) + sink.saturateNext = true + + enqueue(writer, 'ordinary', 'blocker') + enqueue(writer, 'bulk', 'bulk') + enqueue(writer, 'interactive', 'interactive-1') + enqueue(writer, 'ordinary', 'ordinary-1') + enqueue(writer, 'interactive', 'interactive-2') + enqueue(writer, 'ordinary', 'ordinary-2') + enqueue(writer, 'interactive', 'interactive-3') + + sink.drain() + + expect(sink.accepted.map((write) => write.data)).toEqual([ + 'blocker', + 'interactive-1', + 'interactive-2', + 'interactive-3', + 'bulk', + 'ordinary-1', + 'ordinary-2' + ]) + }) + + it('gives ordinary and bulk lanes bounded turns under sustained interactive traffic', () => { + const sink = new FakeSink() + const writer = createWriter(sink) + sink.saturateNext = true + + enqueue(writer, 'ordinary', 'blocker') + enqueue(writer, 'ordinary', 'ordinary') + enqueue(writer, 'bulk', 'bulk') + for (let index = 1; index <= 12; index++) { + enqueue(writer, 'interactive', `interactive-${index}`) + } + + sink.drain() + + expect(sink.accepted.findIndex((write) => write.data === 'ordinary')).toBeLessThanOrEqual(6) + expect(sink.accepted.findIndex((write) => write.data === 'bulk')).toBeLessThanOrEqual(5) + }) + + it('settles callback-less saturated writes when drain fires during registration', () => { + const settled = vi.fn() + const writer = new DispatcherClientWriter( + () => false, + { + waitWriteDrain: (callback) => callback(), + writableHighWaterMark: () => 4096 + }, + vi.fn(), + 64 + ) + + expect(writer.enqueue('ordinary', () => Buffer.from('accepted'), 8, settled)).toBe(true) + expect(settled).toHaveBeenCalledExactlyOnceWith({ ok: true }) + expect(writer.retainedProducerBytes).toBe(0) + }) + + it('bounds queued producer bytes after write(false) settles before drain', () => { + const sink = new FakeSink() + sink.saturateNext = true + const writer = createWriter(sink, vi.fn(), 64) + + enqueue(writer, 'ordinary', 'accepted-before-drain') + sink.accepted[0].settle({ ok: true }) + + for (let index = 0; index < 4; index++) { + expect(writer.enqueue('ordinary', () => Buffer.alloc(16), 16)).toBe(true) + } + expect(writer.enqueue('ordinary', () => Buffer.alloc(1), 1)).toBe(false) + expect(writer.retainedProducerBytes).toBe(64) + expect(sink.accepted).toHaveLength(1) + expect(sink.writableLength + writer.retainedProducerBytes).toBeLessThanOrEqual( + sink.highWaterMark + 64 + ) + + sink.drain() + expect(sink.accepted).toHaveLength(5) + }) + + it('cleans queued and in-flight callbacks once on close', () => { + const sink = new FakeSink() + const writer = createWriter(sink) + sink.saturateNext = true + const first = enqueue(writer, 'ordinary', 'first') + const second = enqueue(writer, 'bulk', 'second') + + writer.close() + sink.drain() + sink.accepted[0].settle({ ok: true }) + + expect(first).toHaveBeenCalledTimes(1) + expect(second).toHaveBeenCalledTimes(1) + expect(writer.retainedProducerBytes).toBe(0) + expect(sink.drainWaiters).toHaveLength(0) + }) + + it('settles the idle fence after accepted writes finish or close cancels them', async () => { + const sink = new FakeSink() + const writer = createWriter(sink) + enqueue(writer, 'ordinary', 'first') + let idle = false + const waiting = writer.waitForIdle().then(() => { + idle = true + }) + + await Promise.resolve() + expect(idle).toBe(false) + sink.accepted[0].settle({ ok: true }) + await waiting + expect(idle).toBe(true) + + enqueue(writer, 'ordinary', 'second') + const canceled = writer.waitForIdle() + writer.close() + await canceled + }) + + it('bounds producer retention and preserves capacity for control', () => { + const sink = new FakeSink() + sink.highWaterMark = 4096 + sink.saturateNext = true + const writer = createWriter(sink, vi.fn(), 2048) + + expect(writer.producerFrameCapacity).toBe(3072) + expect(writer.enqueue('bulk', () => Buffer.alloc(2048), 2048)).toBe(true) + expect(writer.enqueue('bulk', () => Buffer.alloc(1), 1)).toBe(false) + expect(writer.retainedProducerBytes).toBe(2048) + expect(writer.enqueue('control', () => Buffer.from('cancel'), 6)).toBe(true) + + sink.drain() + expect(sink.accepted[1].data).toBe('cancel') + }) + + it('enforces the encoded HWM-minus-reserve limit for splittable producers', () => { + const sink = new FakeSink() + sink.highWaterMark = 4096 + sink.saturateNext = true + const writer = createWriter(sink, vi.fn(), 8192) + + expect(writer.producerFrameCapacity).toBe(3072) + expect(writer.enqueue('bulk', () => Buffer.alloc(3073), 3073)).toBe(false) + expect(writer.enqueue('bulk', () => Buffer.alloc(3072), 3072)).toBe(true) + }) + + it('admits only one indivisible fixed filesystem frame before drain', () => { + const sink = new FakeSink() + sink.highWaterMark = 4096 + sink.saturateNext = true + const writer = createWriter(sink, vi.fn(), 16_384) + const oversized = 'x'.repeat(5000) + + expect(writer.producerFrameCapacity).toBe(3072) + expect(writer.enqueue('fixed-bulk', () => Buffer.from(oversized), 5000)).toBe(true) + expect(writer.enqueue('fixed-bulk', () => Buffer.from(oversized), 5000)).toBe(false) + expect(writer.enqueue('control', () => Buffer.from('cancel'), 6)).toBe(true) + expect(sink.accepted.map((write) => write.data)).toEqual([oversized]) + + sink.writableLength = 0 + sink.accepted[0].settle({ ok: true }) + expect(writer.enqueue('fixed-bulk', () => Buffer.from(oversized), 5000)).toBe(false) + sink.drain() + expect(sink.accepted.map((write) => write.data)).toEqual([oversized, 'cancel']) + }) + + it('gives independent writers progress when one client stays saturated', () => { + const slowSink = new FakeSink() + const fastSink = new FakeSink() + const slow = createWriter(slowSink) + const fast = createWriter(fastSink) + slowSink.saturateNext = true + + enqueue(slow, 'ordinary', 'slow-1') + enqueue(slow, 'ordinary', 'slow-2') + enqueue(fast, 'ordinary', 'fast-1') + enqueue(fast, 'ordinary', 'fast-2') + + expect(slowSink.accepted.map((write) => write.data)).toEqual(['slow-1']) + expect(fastSink.accepted.map((write) => write.data)).toEqual(['fast-1', 'fast-2']) + }) +}) diff --git a/src/relay/dispatcher-client-writer.ts b/src/relay/dispatcher-client-writer.ts new file mode 100644 index 00000000000..1966ce31dfc --- /dev/null +++ b/src/relay/dispatcher-client-writer.ts @@ -0,0 +1,324 @@ +import { + DEFAULT_PRODUCER_QUEUE_MAX_BYTES, + DispatcherWriterAdmission, + onceDispatcherWriterSettlement, + type DispatcherWriterEntry, + type DispatcherWriterLane +} from './dispatcher-writer-admission' +import { DispatcherWriterLaneScheduler } from './dispatcher-writer-lane-scheduler' +import { + DispatcherWriterSink, + type RelayClientSinkOptions, + type RelayClientWrite, + type SinkWriteSettlement +} from './dispatcher-writer-sink' + +export { + DEFAULT_PRODUCER_QUEUE_MAX_BYTES, + DISPATCHER_CONTROL_QUEUE_MAX_BYTES, + relayWriterControlReserve +} from './dispatcher-writer-admission' +export type { + RelayClientSinkOptions, + RelayClientWrite, + SinkWriteSettlement +} from './dispatcher-writer-sink' +export type { DispatcherWriterLane } from './dispatcher-writer-admission' + +export class DispatcherClientWriter { + private readonly admission: DispatcherWriterAdmission + private readonly sink: DispatcherWriterSink + private readonly laneScheduler = new DispatcherWriterLaneScheduler() + private readonly inFlight = new Set() + private readonly settleOnDrain = new Set() + private readonly capacityListeners = new Set<() => void>() + private readonly idleWaiters = new Set<() => void>() + private saturated = false + private drainArmed = false + private removeDrainListener: (() => void) | null = null + private livenessBypassOutstanding = false + private pumping = false + private closed = false + private closeNotified = false + + constructor( + write: RelayClientWrite, + sinkOptions: RelayClientSinkOptions = {}, + private readonly onClosed: (error: Error) => void = () => {}, + producerQueueMaxBytes = DEFAULT_PRODUCER_QUEUE_MAX_BYTES + ) { + this.admission = new DispatcherWriterAdmission(producerQueueMaxBytes) + this.sink = new DispatcherWriterSink(write, sinkOptions) + } + + get retainedProducerBytes(): number { + return this.admission.retainedProducerBytes + } + + get producerFrameCapacity(): number { + return this.sink.producerFrameCapacity + } + + get fixedFrameCapacity(): number { + return this.sink.frameCapacity('fixed-bulk', this.saturated) + } + + canEnqueueProducer(bytes: number): boolean { + return !this.closed && this.admission.canAdmitProducer(bytes, this.producerFrameCapacity) + } + + onCapacity(listener: () => void): () => void { + this.capacityListeners.add(listener) + return () => this.capacityListeners.delete(listener) + } + + waitForIdle(): Promise { + if (this.isIdle()) { + return Promise.resolve() + } + return new Promise((resolve) => this.idleWaiters.add(resolve)) + } + + enqueue( + lane: DispatcherWriterLane, + encode: () => Buffer, + estimatedBytes: number, + onSettled: (result: SinkWriteSettlement) => void = () => {} + ): boolean { + if (this.closed) { + onSettled({ ok: false, error: new Error('Relay writer is closed') }) + return false + } + const entry: DispatcherWriterEntry = { + lane, + encode, + estimatedBytes, + onSettled: onceDispatcherWriterSettlement(onSettled), + settled: false + } + const admission = this.admission.admit(entry, this.sink.frameCapacity(lane, this.saturated)) + if (!admission.accepted) { + if (admission.error) { + entry.onSettled({ ok: false, error: admission.error }) + this.close(admission.error) + } else if (lane === 'liveness') { + entry.onSettled({ ok: true }) + } + return false + } + if (admission.replaced) { + admission.replaced.settled = true + admission.replaced.onSettled({ ok: true }) + } + this.pump() + return true + } + + close(error = new Error('Relay writer closed')): void { + if (this.closed) { + return + } + this.closed = true + this.saturated = false + this.drainArmed = false + this.clearDrainListener() + for (const entry of this.admission.takeQueued()) { + this.releaseEntry(entry, { ok: false, error }) + } + for (const entry of Array.from(this.inFlight)) { + this.releaseEntry(entry, { ok: false, error }) + } + this.settleOnDrain.clear() + this.capacityListeners.clear() + this.notifyIdle() + try { + this.sink.close() + } catch { + // The writer state is already closed. + } + if (!this.closeNotified) { + this.closeNotified = true + this.onClosed(error) + } + } + + private pump(): void { + if (this.pumping || this.closed) { + return + } + this.pumping = true + try { + while (!this.closed) { + const entry = this.selectNext() + if (!entry) { + break + } + this.writeEntry(entry) + if (this.saturated && entry.lane !== 'liveness') { + break + } + } + } finally { + this.pumping = false + } + } + + private selectNext(): DispatcherWriterEntry | undefined { + if (this.saturated) { + if (this.livenessBypassOutstanding) { + return undefined + } + const bypass = this.admission.shift('liveness') + if (bypass) { + this.livenessBypassOutstanding = true + } + return bypass + } + return this.laneScheduler.select( + this.admission, + (entry) => this.canWriteControl(entry), + (entry) => this.canWriteProducer(entry) + ) + } + + private canWriteControl(entry: DispatcherWriterEntry): boolean { + const highWaterMark = this.sink.highWaterMark + if (!Number.isFinite(highWaterMark)) { + return true + } + const writableLength = this.sink.writableLength + return writableLength + entry.estimatedBytes <= highWaterMark || writableLength === 0 + } + + private canWriteProducer(entry: DispatcherWriterEntry): boolean { + const highWaterMark = this.sink.highWaterMark + if (!Number.isFinite(highWaterMark)) { + return true + } + return ( + this.sink.writableLength + entry.estimatedBytes <= this.sink.producerFrameCapacity || + this.sink.writableLength === 0 + ) + } + + private writeEntry(entry: DispatcherWriterEntry): void { + this.laneScheduler.recordWrite(entry.lane) + this.inFlight.add(entry) + let callbackResult: SinkWriteSettlement | undefined + let writeReturned = false + const onWriteSettled = (result: SinkWriteSettlement): void => { + if (!writeReturned) { + callbackResult = result + return + } + this.handleWriteSettlement(entry, result) + } + try { + const accepted = this.sink.write(entry.encode(), onWriteSettled) + writeReturned = true + if (accepted === false) { + this.saturated = true + if (!this.sink.supportsWriteCallback) { + this.settleOnDrain.add(entry) + } + this.armDrain() + } else if (!this.sink.supportsWriteCallback) { + this.handleWriteSettlement(entry, { ok: true }) + } + if (callbackResult) { + this.handleWriteSettlement(entry, callbackResult) + } + } catch (error) { + writeReturned = true + this.handleWriteSettlement(entry, { + ok: false, + error: error instanceof Error ? error : new Error(String(error)) + }) + } + } + + private handleWriteSettlement(entry: DispatcherWriterEntry, result: SinkWriteSettlement): void { + if (!result.ok) { + this.releaseEntry(entry, result) + this.close(result.error) + return + } + this.releaseEntry(entry, result) + if (entry.lane === 'liveness') { + this.livenessBypassOutstanding = false + } + this.notifyCapacity() + this.pump() + } + + private armDrain(): void { + if (this.drainArmed) { + return + } + this.drainArmed = true + try { + const registration = this.sink.registerDrain(() => this.handleDrain()) + if (!registration.registered) { + this.drainArmed = false + } else if (this.drainArmed) { + this.removeDrainListener = registration.remove + } else { + registration.remove() + } + } catch (error) { + this.close(error instanceof Error ? error : new Error(String(error))) + } + } + + private handleDrain(): void { + if (this.closed || !this.drainArmed) { + return + } + this.drainArmed = false + this.clearDrainListener() + this.saturated = false + this.livenessBypassOutstanding = false + for (const entry of Array.from(this.settleOnDrain)) { + this.settleOnDrain.delete(entry) + this.releaseEntry(entry, { ok: true }) + } + this.notifyCapacity() + this.pump() + } + + private releaseEntry(entry: DispatcherWriterEntry, result: SinkWriteSettlement): void { + if (entry.settled) { + return + } + entry.settled = true + this.inFlight.delete(entry) + this.settleOnDrain.delete(entry) + this.admission.release(entry) + entry.onSettled(result) + this.notifyIdle() + } + + private notifyCapacity(): void { + for (const listener of this.capacityListeners) { + listener() + } + } + + private clearDrainListener(): void { + const remove = this.removeDrainListener + this.removeDrainListener = null + remove?.() + } + + private isIdle = (): boolean => this.inFlight.size === 0 && this.admission.queuedEntries === 0 + + private notifyIdle(): void { + if (!this.isIdle()) { + return + } + for (const resolve of Array.from(this.idleWaiters)) { + this.idleWaiters.delete(resolve) + resolve() + } + } +} diff --git a/src/relay/dispatcher-writer-admission.ts b/src/relay/dispatcher-writer-admission.ts new file mode 100644 index 00000000000..a49627d4dcd --- /dev/null +++ b/src/relay/dispatcher-writer-admission.ts @@ -0,0 +1,160 @@ +export type DispatcherWriterLane = + | 'liveness' + | 'control' + | 'legacy-response' + | 'interactive' + | 'ordinary' + | 'fixed-bulk' + | 'bulk' + +export type DispatcherWriterSettlement = { ok: true } | { ok: false; error: Error } + +export type DispatcherWriterEntry = { + lane: DispatcherWriterLane + encode: () => Buffer + estimatedBytes: number + onSettled: (result: DispatcherWriterSettlement) => void + settled: boolean +} + +type AdmissionResult = + | { accepted: true; replaced?: DispatcherWriterEntry } + | { accepted: false; error?: Error } + +const CONTROL_QUEUE_MAX_FRAMES = 256 +export const DISPATCHER_CONTROL_QUEUE_MAX_BYTES = 1024 * 1024 +const LIVENESS_QUEUE_MAX_FRAMES = 2 + +export const DEFAULT_PRODUCER_QUEUE_MAX_BYTES = 2 * 1024 * 1024 + +export function onceDispatcherWriterSettlement( + callback: (result: DispatcherWriterSettlement) => void +): (result: DispatcherWriterSettlement) => void { + let settled = false + return (result) => { + if (settled) { + return + } + settled = true + callback(result) + } +} + +export function relayWriterControlReserve(highWaterMark: number): number { + if (!Number.isFinite(highWaterMark)) { + return 0 + } + return Math.min(64 * 1024, Math.max(1024, Math.floor(highWaterMark / 4))) +} + +export class DispatcherWriterAdmission { + private readonly queues: Record = { + liveness: [], + control: [], + 'legacy-response': [], + interactive: [], + ordinary: [], + 'fixed-bulk': [], + bulk: [] + } + private controlBytes = 0 + private controlFrames = 0 + private producerBytes = 0 + private livenessOutstanding = 0 + + constructor(private readonly producerQueueMaxBytes: number) {} + + get retainedProducerBytes(): number { + return this.producerBytes + } + + get queuedEntries(): number { + return Object.values(this.queues).reduce((total, queue) => total + queue.length, 0) + } + + canAdmitProducer(bytes: number, producerFrameCapacity: number): boolean { + return ( + bytes <= producerFrameCapacity && this.producerBytes + bytes <= this.producerQueueMaxBytes + ) + } + + admit(entry: DispatcherWriterEntry, producerFrameCapacity: number): AdmissionResult { + if (entry.lane === 'liveness') { + return this.admitLiveness(entry) + } + if (entry.lane === 'control') { + return this.admitControl(entry) + } + if (entry.lane === 'legacy-response') { + if (this.producerBytes + entry.estimatedBytes > this.producerQueueMaxBytes) { + return { accepted: false } + } + this.producerBytes += entry.estimatedBytes + this.queues[entry.lane].push(entry) + return { accepted: true } + } + if (entry.lane === 'fixed-bulk' && this.producerBytes > 0) { + return { accepted: false } + } + if (!this.canAdmitProducer(entry.estimatedBytes, producerFrameCapacity)) { + return { accepted: false } + } + this.producerBytes += entry.estimatedBytes + this.queues[entry.lane].push(entry) + return { accepted: true } + } + + peek(lane: DispatcherWriterLane): DispatcherWriterEntry | undefined { + return this.queues[lane][0] + } + + shift(lane: DispatcherWriterLane): DispatcherWriterEntry | undefined { + return this.queues[lane].shift() + } + + takeQueued(): DispatcherWriterEntry[] { + return Object.values(this.queues).flatMap((queue) => queue.splice(0)) + } + + release(entry: DispatcherWriterEntry): void { + if (entry.lane === 'control') { + this.controlFrames-- + this.controlBytes -= entry.estimatedBytes + } else if (entry.lane === 'liveness') { + this.livenessOutstanding-- + } else { + this.producerBytes -= entry.estimatedBytes + } + } + + private admitLiveness(entry: DispatcherWriterEntry): AdmissionResult { + const queue = this.queues.liveness + if (this.livenessOutstanding >= LIVENESS_QUEUE_MAX_FRAMES) { + const replaced = queue.pop() + if (!replaced) { + return { accepted: false } + } + queue.push(entry) + return { accepted: true, replaced } + } + this.livenessOutstanding++ + queue.push(entry) + return { accepted: true } + } + + private admitControl(entry: DispatcherWriterEntry): AdmissionResult { + if ( + this.controlFrames >= CONTROL_QUEUE_MAX_FRAMES || + this.controlBytes + entry.estimatedBytes > DISPATCHER_CONTROL_QUEUE_MAX_BYTES + ) { + return { + accepted: false, + error: new Error('Relay control queue exceeded its bounded capacity') + } + } + this.controlFrames++ + this.controlBytes += entry.estimatedBytes + this.queues.control.push(entry) + return { accepted: true } + } +} diff --git a/src/relay/dispatcher-writer-lane-scheduler.ts b/src/relay/dispatcher-writer-lane-scheduler.ts new file mode 100644 index 00000000000..dba1b254821 --- /dev/null +++ b/src/relay/dispatcher-writer-lane-scheduler.ts @@ -0,0 +1,74 @@ +import type { + DispatcherWriterAdmission, + DispatcherWriterEntry, + DispatcherWriterLane +} from './dispatcher-writer-admission' + +const PRODUCER_WRITES_BEFORE_BULK = 4 +const INTERACTIVE_WRITES_BEFORE_ORDINARY = 4 + +export class DispatcherWriterLaneScheduler { + private producerWritesSinceBulk = 0 + private interactiveWritesSinceOrdinary = 0 + + select( + admission: DispatcherWriterAdmission, + canWriteControl: (entry: DispatcherWriterEntry) => boolean, + canWriteProducer: (entry: DispatcherWriterEntry) => boolean + ): DispatcherWriterEntry | undefined { + const liveness = admission.shift('liveness') + if (liveness) { + return liveness + } + const control = admission.peek('control') + if (control && canWriteControl(control)) { + return admission.shift('control') + } + const legacyResponse = admission.peek('legacy-response') + if (legacyResponse && canWriteControl(legacyResponse)) { + return admission.shift('legacy-response') + } + const bulk = admission.peek('fixed-bulk') ?? admission.peek('bulk') + if ( + bulk && + this.producerWritesSinceBulk >= PRODUCER_WRITES_BEFORE_BULK && + canWriteProducer(bulk) + ) { + return admission.shift(bulk.lane) + } + const ordinary = admission.peek('ordinary') + if ( + ordinary && + this.interactiveWritesSinceOrdinary >= INTERACTIVE_WRITES_BEFORE_ORDINARY && + canWriteProducer(ordinary) + ) { + return admission.shift('ordinary') + } + for (const lane of ['interactive', 'ordinary', 'fixed-bulk', 'bulk'] as const) { + const candidate = admission.peek(lane) + if (candidate && canWriteProducer(candidate)) { + return admission.shift(lane) + } + } + return undefined + } + + recordWrite(lane: DispatcherWriterLane): void { + if (lane === 'bulk' || lane === 'fixed-bulk') { + this.producerWritesSinceBulk = 0 + } else if (lane === 'interactive' || lane === 'ordinary') { + this.producerWritesSinceBulk = Math.min( + PRODUCER_WRITES_BEFORE_BULK, + this.producerWritesSinceBulk + 1 + ) + } + if (lane === 'ordinary') { + this.interactiveWritesSinceOrdinary = 0 + } else if (lane === 'interactive') { + this.interactiveWritesSinceOrdinary = Math.min( + INTERACTIVE_WRITES_BEFORE_ORDINARY, + this.interactiveWritesSinceOrdinary + 1 + ) + } + } +} diff --git a/src/relay/dispatcher-writer-sink.ts b/src/relay/dispatcher-writer-sink.ts new file mode 100644 index 00000000000..ef6c1349134 --- /dev/null +++ b/src/relay/dispatcher-writer-sink.ts @@ -0,0 +1,72 @@ +import { + relayWriterControlReserve, + type DispatcherWriterLane, + type DispatcherWriterSettlement +} from './dispatcher-writer-admission' + +export type SinkWriteSettlement = DispatcherWriterSettlement + +export type RelayClientWrite = ( + data: Buffer, + onSettled: (result: SinkWriteSettlement) => void +) => boolean | void + +export type RelayClientSinkOptions = { + waitWriteDrain?: (callback: () => void) => void | (() => void) + writableLength?: () => number + writableHighWaterMark?: () => number + supportsWriteCallback?: boolean + close?: () => void +} + +export class DispatcherWriterSink { + constructor( + readonly write: RelayClientWrite, + private readonly options: RelayClientSinkOptions + ) {} + + get supportsWriteCallback(): boolean { + return this.options.supportsWriteCallback === true + } + + get writableLength(): number { + const value = this.options.writableLength?.() + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0 + } + + get highWaterMark(): number { + const value = this.options.writableHighWaterMark?.() + return typeof value === 'number' && Number.isFinite(value) && value > 0 + ? value + : Number.POSITIVE_INFINITY + } + + get producerFrameCapacity(): number { + if (!Number.isFinite(this.highWaterMark)) { + return Number.MAX_SAFE_INTEGER + } + return Math.max(0, this.highWaterMark - relayWriterControlReserve(this.highWaterMark)) + } + + frameCapacity(lane: DispatcherWriterLane, producerBlocked = false): number { + if (lane === 'fixed-bulk') { + return !producerBlocked && this.writableLength === 0 ? Number.MAX_SAFE_INTEGER : 0 + } + return this.producerFrameCapacity + } + + registerDrain(callback: () => void): { registered: boolean; remove: () => void } { + if (!this.options.waitWriteDrain) { + return { registered: false, remove: () => {} } + } + const remove = this.options.waitWriteDrain(callback) + return { + registered: true, + remove: typeof remove === 'function' ? remove : () => {} + } + } + + close(): void { + this.options.close?.() + } +} diff --git a/src/relay/dispatcher.test.ts b/src/relay/dispatcher.test.ts index 87f5cb69053..aaa083b7743 100644 --- a/src/relay/dispatcher.test.ts +++ b/src/relay/dispatcher.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import { RelayDispatcher } from './dispatcher' +import { RelayDispatcher, type SinkWriteSettlement } from './dispatcher' import { encodeJsonRpcFrame, encodeKeepAliveFrame, @@ -480,6 +480,111 @@ describe('RelayDispatcher', () => { } }) + it('preserves broadcast order when synchronous settlement re-enters production', () => { + const primary: string[] = [] + const secondary: string[] = [] + const readData = (frame: Buffer): string => { + const message = JSON.parse(decodeFirstFrame(frame).payload.toString()) as JsonRpcNotification + return String(message.params?.data) + } + const orderedDispatcher = new RelayDispatcher((frame) => { + primary.push(readData(frame)) + return true + }) + orderedDispatcher.attachClient((frame) => { + secondary.push(readData(frame)) + return true + }) + let reentered = false + orderedDispatcher.onLegacyPtyCapacity(() => { + if (reentered) { + return + } + reentered = true + orderedDispatcher.tryNotifyPtyData({ id: 'pty-2', data: 'second' }) + }) + + try { + expect(orderedDispatcher.tryNotifyPtyData({ id: 'pty-1', data: 'first' })).toBe(true) + expect(primary).toEqual(['first', 'second']) + expect(secondary).toEqual(['first', 'second']) + } finally { + orderedDispatcher.dispose() + } + }) + + it('does not retry accepted clients when a later broadcast member closes reentrantly', () => { + const primary: string[] = [] + const secondary: string[] = [] + let secondaryId = 0 + const readData = (frame: Buffer): string => { + const message = JSON.parse(decodeFirstFrame(frame).payload.toString()) as JsonRpcNotification + return String(message.params?.data) + } + const dispatcher = new RelayDispatcher((frame) => { + primary.push(readData(frame)) + if (secondaryId !== 0) { + dispatcher.detachClient(secondaryId) + } + return true + }) + secondaryId = dispatcher.attachClient((frame) => { + secondary.push(readData(frame)) + return true + }) + + try { + expect(dispatcher.tryNotifyPtyData({ id: 'pty-1', data: 'once' })).toBe(true) + expect(primary).toEqual(['once']) + expect(secondary).toEqual([]) + } finally { + dispatcher.dispose() + } + }) + + it('keeps a saturated legacy primary as required backpressure', () => { + const callbacks: ((result: SinkWriteSettlement) => void)[] = [] + const legacyDispatcher = new RelayDispatcher( + (_data, settle) => { + callbacks.push(settle) + return false + }, + { + supportsWriteCallback: true, + writableLength: () => 128 * 1024, + writableHighWaterMark: () => 4 * 1024 * 1024 + } + ) + const detached = vi.fn() + legacyDispatcher.onClientDetached(detached) + const payload = 'x'.repeat(128 * 1024) + let admitted = 0 + + try { + while ( + legacyDispatcher.tryNotifyPtyDataToMatchingClients(() => true, { + id: 'pty-1', + data: payload + }) + ) { + admitted++ + } + + expect(admitted).toBeGreaterThan(0) + expect(admitted).toBeLessThan(20) + expect(callbacks).toHaveLength(1) + expect(detached).not.toHaveBeenCalled() + expect( + legacyDispatcher.tryNotifyPtyDataToMatchingClients(() => true, { + id: 'pty-1', + data: payload + }) + ).toBe(false) + } finally { + legacyDispatcher.dispose() + } + }) + describe('notifyBulk (bulk lane backpressure)', () => { it('resolves immediately when the sink accepts the frame', async () => { const frames: Buffer[] = [] @@ -506,7 +611,12 @@ describe('RelayDispatcher', () => { frames.push(Buffer.from(data)) return false }, - { waitWriteDrain: (cb) => drainWaiters.add(cb) } + { + waitWriteDrain: (callback) => { + drainWaiters.add(callback) + return () => drainWaiters.delete(callback) + } + } ) try { let firstSettled = false @@ -533,14 +643,20 @@ describe('RelayDispatcher', () => { } }) - it('interactive notify() frames are not gated behind a stalled bulk lane', async () => { + it('does not write an interactive frame around a saturated bulk write', async () => { const frames: Buffer[] = [] + const drainWaiters = new Set<() => void>() const bulkDispatcher = new RelayDispatcher( (data) => { frames.push(Buffer.from(data)) return false }, - { waitWriteDrain: () => {} } + { + waitWriteDrain: (callback) => { + drainWaiters.add(callback) + return () => drainWaiters.delete(callback) + } + } ) try { void bulkDispatcher.notifyBulk('bulk.event', { seq: 0 }) @@ -549,7 +665,13 @@ describe('RelayDispatcher', () => { expect(frames).toHaveLength(1) bulkDispatcher.notify('pty.data', { id: 'pty-1', data: 'x' }) - expect(frames).toHaveLength(2) + expect(frames).toHaveLength(1) + for (const callback of Array.from(drainWaiters)) { + drainWaiters.delete(callback) + callback() + } + await vi.advanceTimersByTimeAsync(0) + expect(frames.length).toBeGreaterThanOrEqual(2) const msg = JSON.parse( decodeFirstFrame(frames[1]).payload.toString() ) as JsonRpcNotification diff --git a/src/relay/dispatcher.ts b/src/relay/dispatcher.ts index 912b666910f..833081c65fa 100644 --- a/src/relay/dispatcher.ts +++ b/src/relay/dispatcher.ts @@ -13,11 +13,44 @@ import { } from './protocol' import { ClientRequestAborts } from './client-request-aborts' import { MAX_TIMER_DELAY_MS, isSafeTimerDelayMs } from '../shared/timer-delay' +import { + DISPATCHER_CONTROL_QUEUE_MAX_BYTES, + DEFAULT_PRODUCER_QUEUE_MAX_BYTES, + DispatcherClientWriter, + type DispatcherWriterLane, + type RelayClientSinkOptions, + type RelayClientWrite, + type SinkWriteSettlement +} from './dispatcher-client-writer' +import { + LegacyRelayPublicationLedger, + type LegacyPublicationLease +} from './legacy-relay-publication-ledger' + +export type { + RelayClientSinkOptions, + RelayClientWrite, + SinkWriteSettlement +} from './dispatcher-client-writer' export type RequestContext = { clientId: number isStale: () => boolean signal?: AbortSignal + sessionIdentity?: RelayClientSessionIdentity + onResponseSettled?: (handler: (result: SinkWriteSettlement) => void) => void +} + +export type RelayClientSessionIdentity = { + principal: string + authenticated: boolean + allowSessionOwner: boolean + authenticationKind: 'unproved' | 'launch-nonce' | 'endpoint-credential' +} + +export type RelayClientSourceOptions = { + pauseReads?: () => void + resumeReads?: () => void } export type MethodHandler = ( @@ -27,27 +60,16 @@ export type MethodHandler = ( export type NotificationHandler = (params: Record, context: RequestContext) => void -/** Sink write: `false` signals saturation (Node stream semantics); `void`/`true` mean accepted. */ -export type RelayClientWrite = (data: Buffer) => boolean | void - -export type RelayClientSinkOptions = { - /** One-shot: invoke `cb` when the sink can accept more data (drain) or is permanently dead, so waiters never hang. */ - waitWriteDrain?: (cb: () => void) => void -} - type RelayClient = { id: number decoder: FrameDecoder - write: RelayClientWrite - waitWriteDrain?: (cb: () => void) => void - /** Resolvers for bulk sends stalled on sink saturation; flushed so no pump hangs. */ - drainWaiters: Set<() => void> - /** Serializes bulk-lane sends so only one bulk frame is admitted past the sink high-water mark at a time. */ + writer: DispatcherClientWriter bulkChain: Promise nextOutgoingSeq: number highestReceivedSeq: number generation: number closed: boolean + sessionIdentity: RelayClientSessionIdentity } type PendingRelayRequest = { @@ -64,15 +86,26 @@ export class RelayDispatcher { private requestHandlers = new Map() private notificationHandlers = new Map() private readonly requestAborts = new ClientRequestAborts() + private readonly publicationLedger = new LegacyRelayPublicationLedger() private pendingRelayRequests = new Map() private clientDetachListeners = new Set<(clientId: number) => void>() + private disposeListeners = new Set<() => void>() + private legacyCapacityListeners = new Set<() => void>() + private publicationTransactionDepth = 0 + private deferredLegacyCapacity = false + private deferredForcedLegacyCapacity = false private keepaliveTimer: ReturnType | null = null private disposed = false private nextClientId = 1 private nextRequestId = 1 - constructor(write: RelayClientWrite, sinkOptions?: RelayClientSinkOptions) { - this.primaryClient = this.createClient(write, sinkOptions) + constructor( + write: RelayClientWrite, + sinkOptions?: RelayClientSinkOptions, + sessionIdentity?: RelayClientSessionIdentity, + sourceOptions?: RelayClientSourceOptions + ) { + this.primaryClient = this.createClient(write, sinkOptions, sessionIdentity, sourceOptions) this.clients.set(this.primaryClient.id, this.primaryClient) this.startKeepalive() } @@ -81,26 +114,25 @@ export class RelayDispatcher { // Why: the new client's multiplexer restarts at seq=1, so reset seq/decoder state or acks stall and fire a false connection-dead signal. setWrite(write: RelayClientWrite, sinkOptions?: RelayClientSinkOptions): void { this.requestAborts.abortClient(this.primaryClient.id) - this.primaryClient.write = write - this.primaryClient.waitWriteDrain = sinkOptions?.waitWriteDrain - this.primaryClient.closed = false - // Why: the old sink is gone; wake stalled bulk senders to re-evaluate against the new one. - this.flushDrainWaiters(this.primaryClient) + this.primaryClient.closed = true + this.primaryClient.writer.close(new Error('Relay primary sink replaced')) this.resetClient(this.primaryClient) + this.primaryClient.writer = this.createWriter(this.primaryClient, write, sinkOptions) } // Why: mark in-flight requests stale on disconnect so a late pty.spawn/fs.watch can't create unowned remote state. invalidateClient(): void { - this.requestAborts.abortClient(this.primaryClient.id) - this.primaryClient.generation++ - this.primaryClient.closed = true - this.flushDrainWaiters(this.primaryClient) - this.notifyClientDetached(this.primaryClient.id) + this.closeClient(this.primaryClient, new Error('Relay primary client invalidated'), false) } // Why: seq numbers and request ids are per SSH channel, so each attached client needs independent protocol state. - attachClient(write: RelayClientWrite, sinkOptions?: RelayClientSinkOptions): number { - const client = this.createClient(write, sinkOptions) + attachClient( + write: RelayClientWrite, + sinkOptions?: RelayClientSinkOptions, + sessionIdentity?: RelayClientSessionIdentity, + sourceOptions?: RelayClientSourceOptions + ): number { + const client = this.createClient(write, sinkOptions, sessionIdentity, sourceOptions) this.clients.set(client.id, client) return client.id } @@ -110,12 +142,7 @@ export class RelayDispatcher { if (!client || client === this.primaryClient) { return } - this.requestAborts.abortClient(clientId) - client.generation++ - client.closed = true - this.flushDrainWaiters(client) - this.clients.delete(clientId) - this.notifyClientDetached(clientId) + this.closeClient(client, new Error('Relay client detached'), true) } feedClient(clientId: number, data: Buffer): void { @@ -139,6 +166,214 @@ export class RelayDispatcher { return () => this.clientDetachListeners.delete(listener) } + onDisposed(listener: () => void): () => void { + this.disposeListeners.add(listener) + return () => this.disposeListeners.delete(listener) + } + + onLegacyPtyCapacity(listener: () => void): () => void { + this.legacyCapacityListeners.add(listener) + return () => this.legacyCapacityListeners.delete(listener) + } + + get legacyRetentionBelowLowWater(): boolean { + return this.publicationLedger.belowLowWater(this.activeClientKeys()) + } + + writePrimaryBytes(data: Buffer, lane: 'control' | 'ordinary' = 'control'): boolean { + if (this.disposed || this.primaryClient.closed) { + return false + } + return this.primaryClient.writer.enqueue(lane, () => data, data.length) + } + + maxLegacyPtyDataChars( + params: Record, + data: string, + limit = data.length + ): number { + const clients = this.activeClients() + if (clients.length === 0) { + return Math.min(data.length, limit) + } + let low = 0 + let high = Math.min(data.length, limit) + while (low < high) { + const mid = Math.ceil((low + high) / 2) + const msg: JsonRpcNotification = { + jsonrpc: '2.0', + method: 'pty.data', + params: { ...params, data: data.slice(0, mid) } + } + const bytes = this.estimateFrameBytes(msg) + if (clients.every((client) => bytes <= client.writer.producerFrameCapacity)) { + low = mid + } else { + high = mid - 1 + } + } + return low + } + + tryNotifyPtyData( + params: Record, + options: { interactive?: boolean } = {} + ): boolean { + if (this.disposed) { + return false + } + const msg: JsonRpcNotification = { + jsonrpc: '2.0', + method: 'pty.data', + params + } + return this.tryPublishToClients( + this.activeClients(), + msg, + options.interactive ? 'interactive' : 'ordinary' + ) + } + + tryNotifyPtyDataToMatchingClients( + matchesClient: (clientId: number) => boolean, + params: Record, + options: { interactive?: boolean } = {} + ): boolean { + if (this.disposed) { + return false + } + return this.tryPublishToClients( + this.activeClients().filter((client) => matchesClient(client.id)), + { jsonrpc: '2.0', method: 'pty.data', params }, + options.interactive ? 'interactive' : 'ordinary' + ) + } + + projectPtyDataToMatchingClients( + matchesClient: (clientId: number) => boolean, + params: Record, + options: { interactive?: boolean } = {} + ): boolean { + if (this.disposed) { + return false + } + return this.projectToClients( + this.activeClients().filter((client) => matchesClient(client.id)), + { jsonrpc: '2.0', method: 'pty.data', params }, + options.interactive ? 'interactive' : 'ordinary' + ) + } + + tryNotifyPtyDataToClient( + clientId: number, + params: Record, + onSettled: (result: SinkWriteSettlement) => void + ): boolean { + if (this.disposed) { + onSettled({ ok: false, error: new Error('Relay dispatcher is disposed') }) + return false + } + const client = this.clients.get(clientId) + if (!client || client.closed) { + onSettled({ ok: false, error: new Error('Relay client is not connected') }) + return false + } + return this.publishToClient( + client, + { jsonrpc: '2.0', method: 'pty.data', params }, + 'ordinary', + onSettled + ) + } + + tryNotifyPtyExit(params: Record): boolean { + if (this.disposed) { + return false + } + return this.tryPublishToClients( + this.activeClients(), + { + jsonrpc: '2.0', + method: 'pty.exit', + params + }, + 'ordinary' + ) + } + + tryNotifyPtyExitToMatchingClients( + matchesClient: (clientId: number) => boolean, + params: Record + ): boolean { + if (this.disposed) { + return false + } + return this.tryPublishToClients( + this.activeClients().filter((client) => matchesClient(client.id)), + { jsonrpc: '2.0', method: 'pty.exit', params }, + 'ordinary' + ) + } + + projectPtyExitToMatchingClients( + matchesClient: (clientId: number) => boolean, + params: Record + ): boolean { + if (this.disposed) { + return false + } + return this.projectToClients( + this.activeClients().filter((client) => matchesClient(client.id)), + { jsonrpc: '2.0', method: 'pty.exit', params }, + 'ordinary' + ) + } + + tryNotifyPtyExitToClient( + clientId: number, + params: Record, + onSettled: (result: SinkWriteSettlement) => void + ): boolean { + if (this.disposed) { + onSettled({ ok: false, error: new Error('Relay dispatcher is disposed') }) + return false + } + const client = this.clients.get(clientId) + if (!client || client.closed) { + onSettled({ ok: false, error: new Error('Relay client is not connected') }) + return false + } + return this.publishToClient( + client, + { jsonrpc: '2.0', method: 'pty.exit', params }, + 'ordinary', + onSettled + ) + } + + producerDataBudget( + method: string, + paramsWithoutData: Record, + clientId?: number + ): number { + const targets = + clientId === undefined + ? this.activeClients() + : [this.clients.get(clientId)].filter((client): client is RelayClient => !!client) + if (targets.length === 0) { + return Number.MAX_SAFE_INTEGER + } + const emptyFrameBytes = this.estimateFrameBytes({ + jsonrpc: '2.0', + method, + params: { ...paramsWithoutData, data: '' } + }) + return Math.max( + 0, + Math.min(...targets.map((client) => client.writer.producerFrameCapacity - emptyFrameBytes)) + ) + } + feed(data: Buffer): void { this.feedForClient(this.primaryClient, data) } @@ -165,24 +400,75 @@ export class RelayDispatcher { method, ...(params !== undefined ? { params } : {}) } - for (const client of this.clients.values()) { - this.sendFrame(client, msg) - } + this.runPublicationTransaction(() => { + for (const client of this.clients.values()) { + if (client.closed) { + continue + } + if (method === 'pty.replay') { + this.enqueueFrame(client, msg, 'control') + continue + } + if (!this.publishToClient(client, msg, 'ordinary')) { + this.closeClient( + client, + new Error('Relay ordinary publication capacity exceeded'), + client !== this.primaryClient + ) + } + } + }) } notifyClient(clientId: number, method: string, params?: Record): void { + this.tryNotifyClient(clientId, method, params) + } + + tryNotifyClient( + clientId: number, + method: string, + params?: Record, + onSettled: (result: SinkWriteSettlement) => void = () => {} + ): boolean { if (this.disposed) { - return + onSettled({ ok: false, error: new Error('Relay dispatcher is disposed') }) + return false } const client = this.clients.get(clientId) if (!client || client.closed) { + onSettled({ ok: false, error: new Error('Relay client is not connected') }) + return false + } + return this.enqueueFrame( + client, + { + jsonrpc: '2.0', + method, + ...(params !== undefined ? { params } : {}) + }, + 'control', + onSettled + ) + } + + notifyControl(method: string, params?: Record): void { + if (this.disposed) { return } - this.sendFrame(client, { + const msg: JsonRpcNotification = { jsonrpc: '2.0', method, ...(params !== undefined ? { params } : {}) - }) + } + for (const client of this.activeClients()) { + if (!this.enqueueFrame(client, msg, 'control')) { + this.closeClient( + client, + new Error('Relay control publication capacity exceeded'), + client !== this.primaryClient + ) + } + } } /** @@ -213,17 +499,7 @@ export class RelayDispatcher { if (client.closed) { continue } - // Why: encode inside the chain step, not at call time, so sequence numbers match actual write order. - const step = client.bulkChain.then(() => { - if (this.disposed || client.closed) { - return - } - const accepted = this.sendFrame(client, msg) - if (accepted === false) { - return this.waitForClientDrain(client) - } - return undefined - }) + const step = client.bulkChain.then(() => this.publishBulkWhenAvailable(client, msg)) client.bulkChain = step.catch(() => {}) waits.push(step) } @@ -233,35 +509,6 @@ export class RelayDispatcher { return Promise.all(waits).then(() => {}) } - private waitForClientDrain(client: RelayClient): Promise { - if (this.disposed || client.closed || !client.waitWriteDrain) { - return Promise.resolve() - } - return new Promise((resolve) => { - let settled = false - const finish = (): void => { - if (settled) { - return - } - settled = true - client.drainWaiters.delete(finish) - resolve() - } - client.drainWaiters.add(finish) - try { - client.waitWriteDrain!(finish) - } catch { - finish() - } - }) - } - - private flushDrainWaiters(client: RelayClient): void { - for (const waiter of Array.from(client.drainWaiters)) { - waiter() - } - } - requestPrimary( method: string, params?: Record, @@ -315,7 +562,7 @@ export class RelayDispatcher { reject(new Error(`Request "${method}" timed out after ${timeoutMs}ms`)) }, timeoutMs) this.pendingRelayRequests.set(id, { resolve, reject, timer }) - this.sendFrame(client, msg) + this.enqueueFrame(client, msg, 'control') }) } @@ -336,24 +583,48 @@ export class RelayDispatcher { // Why: can't send responses after dispose; abort in-flight work so SSH-side scans/watchers release. this.requestAborts.abortAll() for (const client of this.clients.values()) { - this.flushDrainWaiters(client) + client.closed = true + client.writer.close(new Error('Relay dispatcher disposed')) } + for (const listener of Array.from(this.legacyCapacityListeners)) { + listener() + } + this.legacyCapacityListeners.clear() + for (const listener of Array.from(this.disposeListeners)) { + listener() + } + this.disposeListeners.clear() } - private createClient(write: RelayClientWrite, sinkOptions?: RelayClientSinkOptions): RelayClient { + private createClient( + write: RelayClientWrite, + sinkOptions?: RelayClientSinkOptions, + sessionIdentity?: RelayClientSessionIdentity, + sourceOptions?: RelayClientSourceOptions + ): RelayClient { const id = this.nextClientId++ - const client: RelayClient = { + const client = { id, - decoder: new FrameDecoder((frame) => this.handleFrame(client, frame)), - write, - waitWriteDrain: sinkOptions?.waitWriteDrain, - drainWaiters: new Set(), + decoder: undefined as unknown as FrameDecoder, + writer: undefined as unknown as DispatcherClientWriter, bulkChain: Promise.resolve(), nextOutgoingSeq: 1, highestReceivedSeq: 0, generation: 0, - closed: false - } + closed: false, + sessionIdentity: sessionIdentity ?? { + principal: `unproved:${id}`, + authenticated: false, + allowSessionOwner: false, + authenticationKind: 'unproved' + } + } satisfies RelayClient + client.decoder = new FrameDecoder( + (frame) => this.handleFrame(client, frame), + (error) => this.closeClient(client, error, client !== this.primaryClient), + { pause: sourceOptions?.pauseReads, resume: sourceOptions?.resumeReads } + ) + client.writer = this.createWriter(client, write, sinkOptions) return client } @@ -432,27 +703,70 @@ export class RelayDispatcher { client.id, req.id ) + const responseSettledHandlers = new Set<(result: SinkWriteSettlement) => void>() + let responseSettled = false + const settleResponse = (result: SinkWriteSettlement): void => { + if (responseSettled) { + return + } + responseSettled = true + for (const callback of responseSettledHandlers) { + try { + callback(result) + } catch (err) { + process.stderr.write( + `[relay] Response settlement callback failed: ${err instanceof Error ? err.message : String(err)}\n` + ) + } + } + responseSettledHandlers.clear() + this.requestAborts.delete(abortKey) + } const context: RequestContext = { clientId: client.id, isStale: () => client.generation !== gen || !this.clients.has(client.id) || abortController.signal.aborted, - signal: abortController.signal + signal: abortController.signal, + sessionIdentity: client.sessionIdentity, + onResponseSettled: (handler) => { + if (responseSettled) { + throw new Error('Response settlement callback registered after settlement') + } + responseSettledHandlers.add(handler) + } } try { const result = await handler(req.params ?? {}, context) if (context.isStale()) { + settleResponse({ ok: false, error: new Error('Relay request became stale') }) return } - this.sendResponse(client, req.id, result) + const accepted = this.sendResponse(client, req.id, result, undefined, (settlement) => { + settleResponse( + context.isStale() + ? { ok: false, error: new Error('Relay request became stale') } + : settlement + ) + }) + if (!accepted) { + settleResponse({ ok: false, error: new Error('Relay response was not admitted') }) + } } catch (err) { if (context.isStale()) { + settleResponse({ ok: false, error: new Error('Relay request became stale') }) return } const message = err instanceof Error ? err.message : String(err) const code = (err as { code?: number }).code ?? -32000 - this.sendResponse(client, req.id, undefined, { code, message }) - } finally { - this.requestAborts.delete(abortKey) + const accepted = this.sendResponse(client, req.id, undefined, { code, message }, (result) => { + settleResponse({ + ok: false, + error: result.ok ? new Error(message) : result.error + }) + }) + if (!accepted) { + settleResponse({ ok: false, error: new Error('Relay error response was not admitted') }) + } } } @@ -468,7 +782,11 @@ export class RelayDispatcher { const gen = client.generation handler(notif.params ?? {}, { clientId: client.id, - isStale: () => client.generation !== gen || !this.clients.has(client.id) + isStale: () => client.generation !== gen || !this.clients.has(client.id), + sessionIdentity: client.sessionIdentity, + onResponseSettled: () => { + throw new Error('Notifications do not have response publication fences') + } }) } } @@ -477,26 +795,48 @@ export class RelayDispatcher { client: RelayClient, id: number, result?: unknown, - error?: { code: number; message: string; data?: unknown } - ): void { + error?: { code: number; message: string; data?: unknown }, + onSettled: (result: SinkWriteSettlement) => void = () => {} + ): boolean { const msg: JsonRpcResponse = { jsonrpc: '2.0', id, ...(error ? { error } : { result: result ?? null }) } - this.sendFrame(client, msg) + const estimatedBytes = this.estimateFrameBytes(msg) + const lane = estimatedBytes > DISPATCHER_CONTROL_QUEUE_MAX_BYTES ? 'legacy-response' : 'control' + const accepted = this.enqueueFrame(client, msg, lane, onSettled) + if (!accepted) { + this.closeClient( + client, + new Error( + `Relay response exceeds the bounded ${DEFAULT_PRODUCER_QUEUE_MAX_BYTES}-byte legacy lane` + ), + client !== this.primaryClient + ) + } + return accepted } - private sendFrame( + private enqueueFrame( client: RelayClient, - msg: JsonRpcRequest | JsonRpcResponse | JsonRpcNotification - ): boolean | void { + msg: JsonRpcRequest | JsonRpcResponse | JsonRpcNotification, + lane: DispatcherWriterLane, + onSettled: (result: SinkWriteSettlement) => void = () => {} + ): boolean { if (this.disposed || client.closed) { - return + return false } - const seq = client.nextOutgoingSeq++ - const frame = encodeJsonRpcFrame(msg, seq, client.highestReceivedSeq) - return this.writeFrame(client, frame) + const estimatedBytes = this.estimateFrameBytes(msg) + return client.writer.enqueue( + lane, + () => { + const seq = client.nextOutgoingSeq++ + return encodeJsonRpcFrame(msg, seq, client.highestReceivedSeq) + }, + estimatedBytes, + onSettled + ) } private startKeepalive(): void { @@ -508,31 +848,233 @@ export class RelayDispatcher { if (client.closed) { continue } - const seq = client.nextOutgoingSeq++ - const frame = encodeKeepAliveFrame(seq, client.highestReceivedSeq) - this.writeFrame(client, frame) + client.writer.enqueue( + 'liveness', + () => { + const seq = client.nextOutgoingSeq++ + return encodeKeepAliveFrame(seq, client.highestReceivedSeq) + }, + 13 + ) } }, KEEPALIVE_SEND_MS) // Why: unref so the keepalive interval doesn't pin the event loop and block process exit. this.keepaliveTimer.unref() } - private writeFrame(client: RelayClient, frame: Buffer): boolean | void { - try { - return client.write(frame) - } catch (err) { - client.closed = true - client.generation++ - this.requestAborts.abortClient(client.id) - this.flushDrainWaiters(client) - // Why: frames have no retransmit buffer; detach now so reconnect/PTY-reattach runs instead of waiting the ~20s keepalive timeout. - if (client !== this.primaryClient) { - this.clients.delete(client.id) + private activeClients(): RelayClient[] { + return Array.from(this.clients.values()).filter((client) => !client.closed) + } + + private activeClientKeys(): string[] { + return this.activeClients().map((client) => this.clientKey(client)) + } + + private clientKey(client: RelayClient): string { + return `${client.id}:${client.generation}` + } + + private estimateFrameBytes(msg: JsonRpcRequest | JsonRpcResponse | JsonRpcNotification): number { + return encodeJsonRpcFrame(msg, 0, 0).length + } + + private tryPublishToClients( + clients: readonly RelayClient[], + msg: JsonRpcNotification, + lane: 'interactive' | 'ordinary' | 'bulk' + ): boolean { + return this.runPublicationTransaction(() => { + if (clients.length === 0) { + return true } - this.notifyClientDetached(client.id) - process.stderr.write( - `[relay] Client write failed: ${err instanceof Error ? err.message : String(err)}\n` + const bytes = this.estimateFrameBytes(msg) + if (clients.some((client) => !client.writer.canEnqueueProducer(bytes))) { + return false + } + const leases = this.publicationLedger.tryReserve( + clients.map((client) => ({ clientKey: this.clientKey(client), bytes })) ) + if (!leases) { + return false + } + for (let index = 0; index < clients.length; index++) { + if (!this.enqueueLeasedFrame(clients[index], msg, lane, leases[index])) { + if (this.disposed || clients[index].closed) { + continue + } + for (let remaining = index; remaining < leases.length; remaining++) { + leases[remaining].release() + } + return false + } + } + return true + }) + } + + private projectToClients( + clients: readonly RelayClient[], + msg: JsonRpcNotification, + lane: 'interactive' | 'ordinary' + ): boolean { + return this.runPublicationTransaction(() => { + for (const client of clients) { + if (client.closed || this.publishToClient(client, msg, lane)) { + continue + } + this.closeClient( + client, + new Error('Relay PTY subscriber projection capacity exceeded'), + client !== this.primaryClient + ) + } + return !this.disposed + }) + } + + private publishToClient( + client: RelayClient, + msg: JsonRpcNotification, + lane: 'interactive' | 'ordinary' | 'fixed-bulk' | 'bulk', + onSettled: (result: SinkWriteSettlement) => void = () => {} + ): boolean { + const bytes = this.estimateFrameBytes(msg) + const fixedBlocked = + lane === 'fixed-bulk' && + (client.writer.retainedProducerBytes > 0 || bytes > client.writer.fixedFrameCapacity) + if (fixedBlocked || (lane !== 'fixed-bulk' && !client.writer.canEnqueueProducer(bytes))) { + return false + } + const leases = this.publicationLedger.tryReserve([{ clientKey: this.clientKey(client), bytes }]) + if (!leases) { + return false + } + return this.enqueueLeasedFrame(client, msg, lane, leases[0], onSettled) + } + + private publishBulkWhenAvailable(client: RelayClient, msg: JsonRpcNotification): Promise { + const bytes = this.estimateFrameBytes(msg) + const lane = msg.method === 'fs.streamChunk' ? 'fixed-bulk' : 'bulk' + if (bytes > DEFAULT_PRODUCER_QUEUE_MAX_BYTES) { + return Promise.reject(new Error('Relay bulk frame exceeds sink producer capacity')) + } + if (lane === 'bulk' && bytes > client.writer.producerFrameCapacity) { + return Promise.reject(new Error('Relay bulk frame exceeds sink frame capacity')) + } + return new Promise((resolve, reject) => { + let removeCapacityListener: (() => void) | null = null + const finish = (): void => { + removeCapacityListener?.() + removeCapacityListener = null + } + const tryPublish = (): void => { + if (this.disposed || client.closed) { + finish() + resolve() + return + } + if ( + this.publishToClient(client, msg, lane, (result) => { + finish() + if (result.ok || this.disposed || client.closed) { + resolve() + } else { + reject(result.error) + } + }) + ) { + return + } + if (!removeCapacityListener) { + removeCapacityListener = this.onLegacyPtyCapacity(tryPublish) + } + } + tryPublish() + }) + } + + private enqueueLeasedFrame( + client: RelayClient, + msg: JsonRpcNotification, + lane: 'interactive' | 'ordinary' | 'fixed-bulk' | 'bulk', + lease: LegacyPublicationLease, + onSettled: (result: SinkWriteSettlement) => void = () => {} + ): boolean { + const accepted = this.enqueueFrame(client, msg, lane, (result) => { + lease.release() + onSettled(result) + this.notifyLegacyCapacityIfLow() + }) + if (!accepted) { + lease.release() + this.notifyLegacyCapacityIfLow() + } + return accepted + } + + private createWriter( + client: RelayClient, + write: RelayClientWrite, + sinkOptions?: RelayClientSinkOptions + ): DispatcherClientWriter { + const writer = new DispatcherClientWriter(write, sinkOptions, (error) => { + this.closeClient(client, error, client !== this.primaryClient) + }) + writer.onCapacity(() => this.notifyLegacyCapacityIfLow()) + return writer + } + + private closeClient(client: RelayClient, error: Error, remove: boolean): void { + if (client.closed) { + return + } + client.closed = true + this.requestAborts.abortClient(client.id) + client.writer.close(error) + client.generation++ + if (remove) { + this.clients.delete(client.id) + } + this.notifyClientDetached(client.id) + this.notifyLegacyCapacity(true) + if (!/^Relay (?:primary client invalidated|client detached)$/.test(error.message)) { + process.stderr.write(`[relay] Client write closed: ${error.message}\n`) + } + } + + private notifyLegacyCapacityIfLow(): void { + this.notifyLegacyCapacity(false) + } + + private notifyLegacyCapacity(force: boolean): void { + if (this.publicationTransactionDepth > 0) { + this.deferredForcedLegacyCapacity ||= force + this.deferredLegacyCapacity ||= !force + return + } + if (!force && !this.publicationLedger.belowLowWater(this.activeClientKeys())) { + return + } + for (const listener of this.legacyCapacityListeners) { + listener() + } + } + + private runPublicationTransaction(operation: () => T): T { + this.publicationTransactionDepth++ + try { + return operation() + } finally { + this.publicationTransactionDepth-- + if (this.publicationTransactionDepth === 0) { + const force = this.deferredForcedLegacyCapacity + const low = this.deferredLegacyCapacity + this.deferredForcedLegacyCapacity = false + this.deferredLegacyCapacity = false + if (force || low) { + this.notifyLegacyCapacity(force) + } + } } } diff --git a/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts b/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts index 8cca2787891..98d95fd7f8a 100644 --- a/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts +++ b/src/relay/fs-stream-pty-echo-backpressure.integration.test.ts @@ -28,6 +28,7 @@ import { import { readFileViaStream } from '../main/ssh/ssh-filesystem-stream-reader' import { RelayDispatcher } from './dispatcher' +import type { SinkWriteSettlement } from './dispatcher' import { RelayContext } from './context' import { FsHandler } from './fs-handler' import { STREAM_CHUNK_SIZE } from './protocol' @@ -95,7 +96,10 @@ function createHarness(opts: { congested: boolean }): Harness { onClose: () => {} } - const outQueue: Buffer[] = [] + const outQueue: { + data: Buffer + settle: (result: SinkWriteSettlement) => void + }[] = [] let queuedBytes = 0 const drainWaiters = new Set<() => void>() const fireDrainIfIdle = (): void => { @@ -109,8 +113,8 @@ function createHarness(opts: { congested: boolean }): Harness { } const dispatcher = new RelayDispatcher( - (data: Buffer) => { - outQueue.push(data) + (data: Buffer, settle) => { + outQueue.push({ data, settle }) queuedBytes += data.length if (!opts.congested) { return true @@ -118,6 +122,9 @@ function createHarness(opts: { congested: boolean }): Harness { return queuedBytes < SINK_HIGH_WATER_MARK }, { + supportsWriteCallback: true, + writableLength: () => queuedBytes, + writableHighWaterMark: () => SINK_HIGH_WATER_MARK, waitWriteDrain: (cb: () => void) => { drainWaiters.add(cb) fireDrainIfIdle() @@ -128,11 +135,12 @@ function createHarness(opts: { congested: boolean }): Harness { const deliverAll = (): void => { while (outQueue.length > 0) { - const buf = outQueue.shift()! - queuedBytes -= buf.length + const { data, settle } = outQueue.shift()! + queuedBytes -= data.length for (const cb of clientDataCallbacks) { - cb(buf) + cb(data) } + settle({ ok: true }) } fireDrainIfIdle() } @@ -207,7 +215,7 @@ describe('fs.readFileStream vs pty.data echo head-of-line blocking', () => { // The echo must not sit behind an unbounded chunk backlog: at most one // in-flight bulk frame (the write that saturated the sink) plus slack. - expect(queuedBytesAheadOfEcho).toBeLessThan(2 * FRAMED_CHUNK_BYTES) + expect(queuedBytesAheadOfEcho).toBeLessThan(FRAMED_CHUNK_BYTES) // Un-congest: the stream must still complete with intact content. harness.startAutoDeliver() diff --git a/src/relay/git-response-pty-echo-backpressure.integration.test.ts b/src/relay/git-response-pty-echo-backpressure.integration.test.ts index 09e6bf38192..4f3eee4a87a 100644 --- a/src/relay/git-response-pty-echo-backpressure.integration.test.ts +++ b/src/relay/git-response-pty-echo-backpressure.integration.test.ts @@ -28,6 +28,7 @@ import { import { requestGitStreamable } from '../main/ssh/ssh-git-response-stream-reader' import { RelayDispatcher } from './dispatcher' +import type { SinkWriteSettlement } from './dispatcher' import { RelayContext } from './context' import { GitHandler } from './git-handler' import { GIT_RESPONSE_CHUNK_SIZE } from './protocol' @@ -91,7 +92,10 @@ function createHarness(opts: { congested: boolean }): Harness { onClose: () => {} } - const outQueue: Buffer[] = [] + const outQueue: { + data: Buffer + settle: (result: SinkWriteSettlement) => void + }[] = [] let queuedBytes = 0 const drainWaiters = new Set<() => void>() const fireDrainIfIdle = (): void => { @@ -105,8 +109,8 @@ function createHarness(opts: { congested: boolean }): Harness { } const dispatcher = new RelayDispatcher( - (data: Buffer) => { - outQueue.push(data) + (data: Buffer, settle) => { + outQueue.push({ data, settle }) queuedBytes += data.length if (!opts.congested) { return true @@ -114,6 +118,9 @@ function createHarness(opts: { congested: boolean }): Harness { return queuedBytes < SINK_HIGH_WATER_MARK }, { + supportsWriteCallback: true, + writableLength: () => queuedBytes, + writableHighWaterMark: () => SINK_HIGH_WATER_MARK, waitWriteDrain: (cb: () => void) => { drainWaiters.add(cb) fireDrainIfIdle() @@ -124,11 +131,12 @@ function createHarness(opts: { congested: boolean }): Harness { const deliverAll = (): void => { while (outQueue.length > 0) { - const buf = outQueue.shift()! - queuedBytes -= buf.length + const { data, settle } = outQueue.shift()! + queuedBytes -= data.length for (const cb of clientDataCallbacks) { - cb(buf) + cb(data) } + settle({ ok: true }) } fireDrainIfIdle() } diff --git a/src/relay/git-response-stream-ownership.test.ts b/src/relay/git-response-stream-ownership.test.ts index c6340265238..66d7ead7935 100644 --- a/src/relay/git-response-stream-ownership.test.ts +++ b/src/relay/git-response-stream-ownership.test.ts @@ -65,4 +65,50 @@ describe('GitResponseStreamRegistry client ownership', () => { expect(notifyBulk.mock.calls[0]?.[0]).toBe('git.responseChunk') expect(notifyBulk.mock.calls[1]?.[0]).toBe('git.responseError') }) + + it('uses encoded producer capacity without collapsing chunks during saturation', async () => { + let releaseFirst!: () => void + const firstWrite = new Promise((resolve) => { + releaseFirst = resolve + }) + const notifyBulk = vi + .fn() + .mockImplementationOnce(() => firstWrite) + .mockResolvedValue(undefined) + const producerDataBudget = vi.fn(() => 8) + const dispatcher = { + notifyBulk, + producerDataBudget + } as unknown as RelayDispatcher + const registry = new GitResponseStreamRegistry() + registries.push(registry) + const ownerClientId = 7 + + const marker = registry.startStream(Buffer.alloc(12), dispatcher, { + clientId: ownerClientId, + isStale: () => false + }) + const streamId = marker.__orcaGitResponseStream.streamId + await flushPump() + expect(notifyBulk).toHaveBeenCalledTimes(1) + + registry.recordAck(streamId, 1_000, ownerClientId) + releaseFirst() + await flushPump() + await flushPump() + + expect(producerDataBudget).toHaveBeenCalledExactlyOnceWith( + 'git.responseChunk', + { streamId, seq: 12 }, + ownerClientId + ) + expect(marker.__orcaGitResponseStream.chunkCount).toBe(2) + expect(notifyBulk).toHaveBeenCalledTimes(3) + const chunks = notifyBulk.mock.calls + .filter(([method]) => method === 'git.responseChunk') + .map(([, params]) => params as { data: string }) + expect(chunks).toHaveLength(2) + expect(chunks.every(({ data }) => data.length <= 8)).toBe(true) + expect(chunks.map(({ data }) => Buffer.from(data, 'base64').length)).toEqual([6, 6]) + }) }) diff --git a/src/relay/git-response-stream.ts b/src/relay/git-response-stream.ts index bced3b47e46..3ccbd66ee8d 100644 --- a/src/relay/git-response-stream.ts +++ b/src/relay/git-response-stream.ts @@ -22,10 +22,10 @@ type GitResponseStreamEntry = { /** Serialized git responses are chunked as base64 so multi-byte UTF-8 * sequences never split across a chunk boundary (the client concatenates the * decoded bytes and parses once). */ -function encodeChunks(payload: Buffer): string[] { +function encodeChunks(payload: Buffer, chunkBytes = GIT_RESPONSE_CHUNK_SIZE): string[] { const chunks: string[] = [] - for (let offset = 0; offset < payload.length; offset += GIT_RESPONSE_CHUNK_SIZE) { - chunks.push(payload.subarray(offset, offset + GIT_RESPONSE_CHUNK_SIZE).toString('base64')) + for (let offset = 0; offset < payload.length; offset += chunkBytes) { + chunks.push(payload.subarray(offset, offset + chunkBytes).toString('base64')) } return chunks } @@ -116,7 +116,18 @@ export class GitResponseStreamRegistry { context: RequestContext ): GitResponseStreamMarker { const streamId = this.register(context.clientId) - const chunks = encodeChunks(payload) + const base64Budget = + dispatcher.producerDataBudget?.( + 'git.responseChunk', + { streamId, seq: payload.length }, + context.clientId + ) ?? Number.MAX_SAFE_INTEGER + const sinkChunkBytes = Math.floor(Math.max(0, base64Budget) / 4) * 3 + if (sinkChunkBytes === 0) { + this.streams.delete(streamId) + throw new Error('Git response stream has no encoded producer capacity') + } + const chunks = encodeChunks(payload, Math.min(GIT_RESPONSE_CHUNK_SIZE, sinkChunkBytes)) // Why: kick the pump off the response task so the client sees the sentinel // (and can subscribe/reassemble) before the first chunk frame arrives. setImmediate(() => { diff --git a/src/relay/legacy-relay-publication-ledger.test.ts b/src/relay/legacy-relay-publication-ledger.test.ts new file mode 100644 index 00000000000..04d64a55407 --- /dev/null +++ b/src/relay/legacy-relay-publication-ledger.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { LegacyRelayPublicationLedger } from './legacy-relay-publication-ledger' + +describe('LegacyRelayPublicationLedger', () => { + it('caps each client and aggregate retained bytes', () => { + const ledger = new LegacyRelayPublicationLedger({ + clientHighBytes: 10, + clientLowBytes: 4, + relayHighBytes: 15, + relayLowBytes: 8 + }) + const first = ledger.tryReserve([ + { clientKey: 'a', bytes: 7 }, + { clientKey: 'b', bytes: 7 } + ]) + expect(first).not.toBeNull() + expect(ledger.retainedBytes).toBe(14) + expect(ledger.tryReserve([{ clientKey: 'a', bytes: 4 }])).toBeNull() + expect(ledger.tryReserve([{ clientKey: 'c', bytes: 2 }])).toBeNull() + + first?.[0].release() + expect(ledger.retainedBytes).toBe(7) + expect(ledger.tryReserve([{ clientKey: 'c', bytes: 2 }])).not.toBeNull() + }) + + it('releases leases exactly once and applies both low waters', () => { + const ledger = new LegacyRelayPublicationLedger({ + clientHighBytes: 10, + clientLowBytes: 3, + relayHighBytes: 20, + relayLowBytes: 5 + }) + const leases = ledger.tryReserve([ + { clientKey: 'a', bytes: 4 }, + { clientKey: 'b', bytes: 4 } + ]) + expect(ledger.belowLowWater()).toBe(false) + + leases?.[0].release() + leases?.[0].release() + expect(ledger.retainedBytes).toBe(4) + expect(ledger.belowLowWater()).toBe(false) + + leases?.[1].release() + expect(ledger.retainedBytes).toBe(0) + expect(ledger.belowLowWater()).toBe(true) + }) +}) diff --git a/src/relay/legacy-relay-publication-ledger.ts b/src/relay/legacy-relay-publication-ledger.ts new file mode 100644 index 00000000000..f7002756bc2 --- /dev/null +++ b/src/relay/legacy-relay-publication-ledger.ts @@ -0,0 +1,104 @@ +export const LEGACY_CLIENT_RETAINED_BYTES_HIGH = 2 * 1024 * 1024 +export const LEGACY_CLIENT_RETAINED_BYTES_LOW = 1024 * 1024 +export const LEGACY_RELAY_RETAINED_BYTES_HIGH = 32 * 1024 * 1024 +export const LEGACY_RELAY_RETAINED_BYTES_LOW = 24 * 1024 * 1024 + +export type LegacyPublicationLease = { + clientKey: string + bytes: number + release: () => void +} + +type LegacyPublicationLimits = { + clientHighBytes?: number + clientLowBytes?: number + relayHighBytes?: number + relayLowBytes?: number +} + +export class LegacyRelayPublicationLedger { + private readonly clientBytes = new Map() + private aggregateBytes = 0 + + readonly clientHighBytes: number + readonly clientLowBytes: number + readonly relayHighBytes: number + readonly relayLowBytes: number + + constructor(limits: LegacyPublicationLimits = {}) { + this.clientHighBytes = limits.clientHighBytes ?? LEGACY_CLIENT_RETAINED_BYTES_HIGH + this.clientLowBytes = limits.clientLowBytes ?? LEGACY_CLIENT_RETAINED_BYTES_LOW + this.relayHighBytes = limits.relayHighBytes ?? LEGACY_RELAY_RETAINED_BYTES_HIGH + this.relayLowBytes = limits.relayLowBytes ?? LEGACY_RELAY_RETAINED_BYTES_LOW + } + + get retainedBytes(): number { + return this.aggregateBytes + } + + retainedBytesFor(clientKey: string): number { + return this.clientBytes.get(clientKey) ?? 0 + } + + tryReserve( + memberships: readonly { clientKey: string; bytes: number }[] + ): LegacyPublicationLease[] | null { + let aggregateAdded = 0 + const additions = new Map() + for (const membership of memberships) { + if (!Number.isSafeInteger(membership.bytes) || membership.bytes < 0) { + return null + } + aggregateAdded += membership.bytes + additions.set( + membership.clientKey, + (additions.get(membership.clientKey) ?? 0) + membership.bytes + ) + } + if (this.aggregateBytes + aggregateAdded > this.relayHighBytes) { + return null + } + for (const [clientKey, bytes] of additions) { + if ((this.clientBytes.get(clientKey) ?? 0) + bytes > this.clientHighBytes) { + return null + } + } + this.aggregateBytes += aggregateAdded + for (const [clientKey, bytes] of additions) { + this.clientBytes.set(clientKey, (this.clientBytes.get(clientKey) ?? 0) + bytes) + } + return memberships.map(({ clientKey, bytes }) => { + let released = false + return { + clientKey, + bytes, + release: () => { + if (released) { + return + } + released = true + this.release(clientKey, bytes) + } + } + }) + } + + belowLowWater(clientKeys?: readonly string[]): boolean { + if (this.aggregateBytes > this.relayLowBytes) { + return false + } + const keys = clientKeys ?? Array.from(this.clientBytes.keys()) + return keys.every((clientKey) => (this.clientBytes.get(clientKey) ?? 0) <= this.clientLowBytes) + } + + private release(clientKey: string, bytes: number): void { + const current = this.clientBytes.get(clientKey) ?? 0 + const next = Math.max(0, current - bytes) + this.aggregateBytes = Math.max(0, this.aggregateBytes - Math.min(bytes, current)) + if (next === 0) { + this.clientBytes.delete(clientKey) + } else { + this.clientBytes.set(clientKey, next) + } + } +} diff --git a/src/relay/protocol-backpressure.test.ts b/src/relay/protocol-backpressure.test.ts new file mode 100644 index 00000000000..fa4724ff960 --- /dev/null +++ b/src/relay/protocol-backpressure.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, it, vi } from 'vitest' +import { + FrameDecoder, + FrameDecoderContinuationError, + FRAME_DECODER_MAX_RETAINED_BYTES, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + MessageType, + encodeFrame, + type DecodedFrame +} from './protocol' + +function createScheduler(): { + schedule: (callback: () => void) => number + cancel: (handle: unknown) => void + runNext: () => void + pending: () => number +} { + let nextHandle = 1 + const callbacks = new Map void>() + return { + schedule: (callback) => { + const handle = nextHandle++ + callbacks.set(handle, callback) + return handle + }, + cancel: (handle) => callbacks.delete(handle as number), + runNext: () => { + const entry = callbacks.entries().next().value as [number, () => void] | undefined + if (!entry) { + throw new Error('No decoder continuation scheduled') + } + callbacks.delete(entry[0]) + entry[1]() + }, + pending: () => callbacks.size + } +} + +function frame(id: number, payload = `${id}`): Buffer { + return encodeFrame(MessageType.Regular, id, 0, Buffer.from(payload)) +} + +describe('relay FrameDecoder bounded turns', () => { + it('retains at most one maximum frame plus one MiB of partial input', () => { + expect(FRAME_DECODER_MAX_RETAINED_BYTES).toBe(MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024) + const maximumFrame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(MAX_MESSAGE_SIZE)) + const acceptedError = vi.fn() + const accepted = new FrameDecoder(vi.fn(), acceptedError) + + accepted.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024)])) + + expect(acceptedError).not.toHaveBeenCalled() + expect(accepted.drain()).toHaveLength(1024 * 1024) + + const excessError = vi.fn() + const excess = new FrameDecoder(vi.fn(), excessError) + excess.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024 + 1)])) + + expect(excessError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('retained-input') }) + ) + expect(excess.drain()).toHaveLength(0) + }) + + it('fails closed when one delivered chunk exceeds retained-input capacity', () => { + const onError = vi.fn() + const decoder = new FrameDecoder(vi.fn(), onError) + + decoder.feed(Buffer.alloc(FRAME_DECODER_MAX_RETAINED_BYTES + 1)) + + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining('retained-input') }) + ) + expect(decoder.drain()).toHaveLength(0) + }) + + it('emits the first frame synchronously and preserves order through self-pause', () => { + const scheduler = createScheduler() + const seen: number[] = [] + let decoder: FrameDecoder + const pause = vi.fn(() => decoder.feed(frame(4))) + const resume = vi.fn() + decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, { + maxFramesPerTurn: 1, + schedule: scheduler.schedule, + cancelScheduled: scheduler.cancel, + pause, + resume + }) + + decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)])) + + expect(seen).toEqual([1]) + expect(pause).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(1) + + scheduler.runNext() + scheduler.runNext() + scheduler.runNext() + + expect(seen).toEqual([1, 2, 3, 4]) + expect(pause).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(0) + }) + + it('bounds decoded bytes and time independently from the frame count', () => { + const byteScheduler = createScheduler() + const byteSeen: number[] = [] + const first = frame(1, 'one') + const second = frame(2, 'two') + const byteDecoder = new FrameDecoder((decoded) => byteSeen.push(decoded.id), undefined, { + maxFramesPerTurn: 64, + maxBytesPerTurn: first.length, + schedule: byteScheduler.schedule, + cancelScheduled: byteScheduler.cancel + }) + + byteDecoder.feed(Buffer.concat([first, second])) + expect(byteSeen).toEqual([1]) + byteScheduler.runNext() + expect(byteSeen).toEqual([1, 2]) + + const timeScheduler = createScheduler() + const timeSeen: number[] = [] + let nowCalls = 0 + const timeDecoder = new FrameDecoder((decoded) => timeSeen.push(decoded.id), undefined, { + maxFramesPerTurn: 64, + maxBytesPerTurn: MAX_MESSAGE_SIZE + HEADER_LENGTH, + maxTurnMs: 4, + now: () => (nowCalls++ === 0 ? 0 : 5), + schedule: timeScheduler.schedule, + cancelScheduled: timeScheduler.cancel + }) + + timeDecoder.feed(Buffer.concat([frame(3), frame(4)])) + expect(timeSeen).toEqual([3]) + timeScheduler.runNext() + expect(timeSeen).toEqual([3, 4]) + }) + + it('releases its pause epoch when continuation scheduling throws', () => { + const pause = vi.fn() + const resume = vi.fn() + const decoder = new FrameDecoder(() => {}, undefined, { + maxFramesPerTurn: 1, + pause, + resume, + schedule: () => { + throw new Error('scheduler unavailable') + } + }) + + expect(() => decoder.feed(Buffer.concat([frame(1), frame(2)]))).toThrow('scheduler unavailable') + expect(pause).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + decoder.reset() + expect(resume).toHaveBeenCalledTimes(1) + }) + + it('contains a throwing continuation, resets residue, and reports one typed error', () => { + const scheduler = createScheduler() + const seen: number[] = [] + const onError = vi.fn() + const pause = vi.fn() + const resume = vi.fn() + const decoder = new FrameDecoder( + (decoded) => { + if (decoded.id === 2) { + throw new Error('frame owner failed') + } + seen.push(decoded.id) + }, + onError, + { + maxFramesPerTurn: 1, + schedule: scheduler.schedule, + cancelScheduled: scheduler.cancel, + pause, + resume + } + ) + + decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)])) + expect(() => scheduler.runNext()).not.toThrow() + + expect(seen).toEqual([1]) + expect(onError).toHaveBeenCalledExactlyOnceWith(expect.any(FrameDecoderContinuationError)) + expect(onError.mock.calls[0]?.[0]).toMatchObject({ + name: 'FrameDecoderContinuationError', + cause: expect.objectContaining({ message: 'frame owner failed' }) + }) + expect(pause).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(0) + expect(decoder.drain()).toHaveLength(0) + + decoder.feed(frame(4)) + expect(seen).toEqual([1, 4]) + }) + + it('keeps reads active for partial frames and incrementally discards oversized payloads', () => { + const errors: Error[] = [] + const seen: DecodedFrame[] = [] + const pause = vi.fn() + const decoder = new FrameDecoder( + (decoded) => seen.push(decoded), + (error) => errors.push(error), + { pause } + ) + const valid = frame(2, 'complete') + + decoder.feed(valid.subarray(0, HEADER_LENGTH + 2)) + expect(seen).toHaveLength(0) + expect(pause).not.toHaveBeenCalled() + decoder.feed(valid.subarray(HEADER_LENGTH + 2)) + expect(seen.map(({ id }) => id)).toEqual([2]) + + const oversizedHeader = Buffer.alloc(HEADER_LENGTH) + oversizedHeader[0] = MessageType.Regular + oversizedHeader.writeUInt32BE(3, 1) + oversizedHeader.writeUInt32BE(MAX_MESSAGE_SIZE + 1, 9) + decoder.feed(Buffer.concat([oversizedHeader, Buffer.alloc(32)])) + + expect(errors).toHaveLength(1) + expect(pause).not.toHaveBeenCalled() + decoder.reset() + decoder.feed(frame(4, 'after-reset')) + expect(seen.map(({ id }) => id)).toEqual([2, 4]) + }) + + it('resynchronizes after an oversized payload across bounded discard turns', () => { + const scheduler = createScheduler() + const errors: Error[] = [] + const seen: number[] = [] + const decoder = new FrameDecoder( + (decoded) => seen.push(decoded.id), + (error) => errors.push(error), + { schedule: scheduler.schedule, cancelScheduled: scheduler.cancel } + ) + const oversizedLength = MAX_MESSAGE_SIZE + 1 + const header = Buffer.alloc(HEADER_LENGTH) + header[0] = MessageType.Regular + header.writeUInt32BE(1, 1) + header.writeUInt32BE(oversizedLength, 9) + + decoder.feed(Buffer.concat([header, Buffer.alloc(oversizedLength), frame(2, 'after')])) + + expect(errors).toHaveLength(1) + expect(seen).toHaveLength(0) + expect(scheduler.pending()).toBe(1) + scheduler.runNext() + expect(seen).toEqual([2]) + }) + + it('drain and reset cancel continuation ownership without replaying residue', () => { + const scheduler = createScheduler() + const seen: number[] = [] + const resume = vi.fn() + const cancel = vi.fn(scheduler.cancel) + const decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, { + maxFramesPerTurn: 1, + schedule: scheduler.schedule, + cancelScheduled: cancel, + resume + }) + const second = frame(2, 'residue') + + decoder.feed(Buffer.concat([frame(1), second])) + const residue = decoder.drain() + + expect(seen).toEqual([1]) + expect(residue.equals(second)).toBe(true) + expect(cancel).toHaveBeenCalledTimes(1) + expect(resume).toHaveBeenCalledTimes(1) + expect(scheduler.pending()).toBe(0) + + decoder.feed(Buffer.concat([frame(3), frame(4)])) + decoder.reset() + expect(cancel).toHaveBeenCalledTimes(2) + expect(scheduler.pending()).toBe(0) + + decoder.feed(frame(5)) + expect(seen).toEqual([1, 3, 5]) + }) +}) diff --git a/src/relay/protocol.ts b/src/relay/protocol.ts index 617ab84e679..0fbd85cc8fb 100644 --- a/src/relay/protocol.ts +++ b/src/relay/protocol.ts @@ -1,12 +1,32 @@ // Self-contained relay protocol — mirrors src/main/ssh/relay-protocol.ts // but has no Electron dependencies. Deployed standalone to remote hosts. +import { + FrameDecoder, + FrameDecoderContinuationError, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + FRAME_DECODER_MAX_FRAMES_PER_TURN, + FRAME_DECODER_MAX_BYTES_PER_TURN, + FRAME_DECODER_MAX_TURN_MS, + FRAME_DECODER_MAX_RETAINED_BYTES +} from './relay-frame-decoder' + +export { + FrameDecoder, + FrameDecoderContinuationError, + HEADER_LENGTH, + MAX_MESSAGE_SIZE, + FRAME_DECODER_MAX_FRAMES_PER_TURN, + FRAME_DECODER_MAX_BYTES_PER_TURN, + FRAME_DECODER_MAX_TURN_MS, + FRAME_DECODER_MAX_RETAINED_BYTES +} +export type { DecodedFrame, FrameDecoderOptions } from './relay-frame-decoder' + export const RELAY_VERSION = '0.1.0' export const RELAY_SENTINEL = `ORCA-RELAY v${RELAY_VERSION} READY\n` -export const HEADER_LENGTH = 13 -export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 - export const MessageType = { Regular: 1, Handshake: 2, @@ -18,7 +38,7 @@ export const MessageType = { // to refuse mismatched-version --connect bridges that would otherwise drive a // stale daemon. export type HandshakeMessage = - | { type: 'orca-relay-handshake'; version: string } + | { type: 'orca-relay-handshake'; version: string; endpointCredential?: string } | { type: 'orca-relay-handshake-ok'; version: string } | { type: 'orca-relay-handshake-mismatch'; expected: string; got: string } @@ -107,13 +127,6 @@ export type JsonRpcNotification = { export type JsonRpcMessage = JsonRpcRequest | JsonRpcResponse | JsonRpcNotification -export type DecodedFrame = { - type: number - id: number - ack: number - payload: Buffer -} - export function encodeFrame( type: number, id: number, @@ -140,148 +153,6 @@ export function encodeKeepAliveFrame(id: number, ack: number): Buffer { return encodeFrame(MessageType.KeepAlive, id, ack, Buffer.alloc(0)) } -export class FrameDecoder { - // Why: feed() sits on the hot receive path. Rebuilding one contiguous - // buffer per feed (Buffer.concat) re-copies every already-buffered byte for - // each incoming chunk — O(n²) per large frame. A chunk list assembles each - // frame exactly once instead. - private chunks: Buffer[] = [] - private bufferedLength = 0 - private onFrame: (frame: DecodedFrame) => void - private onError: ((err: Error) => void) | null - - constructor(onFrame: (frame: DecodedFrame) => void, onError?: (err: Error) => void) { - this.onFrame = onFrame - this.onError = onError ?? null - } - - feed(chunk: Buffer | Uint8Array): void { - const buf = Buffer.isBuffer(chunk) - ? chunk - : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) - if (buf.length > 0) { - this.chunks.push(buf) - this.bufferedLength += buf.length - } - - while (this.bufferedLength >= HEADER_LENGTH) { - const header = this.peekBytes(HEADER_LENGTH) - const length = header.readUInt32BE(9) - const totalLength = HEADER_LENGTH + length - - if (this.bufferedLength < totalLength) { - // Not fully received yet (also holds oversized frames until they can - // be skipped whole, keeping the decoder synchronized). - break - } - - if (length > MAX_MESSAGE_SIZE) { - // Why: Throwing here would leave the buffer in a partially consumed - // state — subsequent feed() calls would try to parse the leftover - // payload bytes as a new header, corrupting every future frame. - // Instead we skip the entire oversized frame so the decoder stays - // synchronized with the stream. - this.discardBytes(totalLength) - const err = new Error(`Frame payload too large: ${length} bytes — discarded`) - if (this.onError) { - this.onError(err) - } else { - process.stderr.write(`[relay] ${err.message}\n`) - } - continue - } - - const framed = this.takeBytes(totalLength) - const frame: DecodedFrame = { - type: framed[0], - id: framed.readUInt32BE(1), - ack: framed.readUInt32BE(5), - payload: framed.subarray(HEADER_LENGTH, totalLength) - } - this.onFrame(frame) - } - } - - reset(): void { - this.chunks = [] - this.bufferedLength = 0 - } - - // Why: at the handshake → dispatcher transition, the next consumer must - // pick up any bytes that arrived in the same TCP chunk as the handshake - // frame. This returns and clears the decoder's internal residue so the - // caller can hand it to the dispatcher (or stdout pipe) without loss. - drain(): Buffer { - const out = - this.chunks.length === 1 ? this.chunks[0] : Buffer.concat(this.chunks, this.bufferedLength) - this.reset() - return out - } - - /** View of the first `count` buffered bytes without consuming them. */ - private peekBytes(count: number): Buffer { - const first = this.chunks[0] - if (first.length >= count) { - return first - } - const out = Buffer.allocUnsafe(count) - let copied = 0 - for (const part of this.chunks) { - copied += part.copy(out, copied, 0, Math.min(part.length, count - copied)) - if (copied >= count) { - break - } - } - return out - } - - /** Consume and return the first `count` buffered bytes (single copy). */ - private takeBytes(count: number): Buffer { - const first = this.chunks[0] - if (first.length === count) { - this.chunks.shift() - this.bufferedLength -= count - return first - } - if (first.length > count) { - this.chunks[0] = first.subarray(count) - this.bufferedLength -= count - return first.subarray(0, count) - } - const out = Buffer.allocUnsafe(count) - let copied = 0 - while (copied < count) { - const part = this.chunks[0] - const take = Math.min(part.length, count - copied) - part.copy(out, copied, 0, take) - copied += take - if (take === part.length) { - this.chunks.shift() - } else { - this.chunks[0] = part.subarray(take) - } - } - this.bufferedLength -= count - return out - } - - /** Consume the first `count` buffered bytes without assembling them. */ - private discardBytes(count: number): void { - let remaining = count - while (remaining > 0) { - const part = this.chunks[0] - if (part.length <= remaining) { - this.chunks.shift() - remaining -= part.length - } else { - this.chunks[0] = part.subarray(remaining) - remaining = 0 - } - } - this.bufferedLength -= count - } -} - export function parseJsonRpcMessage(payload: Buffer): JsonRpcMessage { const text = payload.toString('utf-8') const msg = JSON.parse(text) as JsonRpcMessage diff --git a/src/relay/pty-handler-output-drain-differential.test.ts b/src/relay/pty-handler-output-drain-differential.test.ts index b2527e92ec4..22ba265cd10 100644 --- a/src/relay/pty-handler-output-drain-differential.test.ts +++ b/src/relay/pty-handler-output-drain-differential.test.ts @@ -9,7 +9,9 @@ const { mockPtySpawn, mockPtyInstance } = vi.hoisted(() => ({ write: vi.fn(), resize: vi.fn(), kill: vi.fn(), - clear: vi.fn() + clear: vi.fn(), + pause: vi.fn(), + resume: vi.fn() } })) diff --git a/src/relay/pty-handler-source-publication.test.ts b/src/relay/pty-handler-source-publication.test.ts new file mode 100644 index 00000000000..a99610671ec --- /dev/null +++ b/src/relay/pty-handler-source-publication.test.ts @@ -0,0 +1,344 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PTY_STARTUP_INGRESS_VERSION } from '../shared/pty-startup-ingress' +import { + RelayDispatcher, + type RelayClientSessionIdentity, + type SinkWriteSettlement +} from './dispatcher' +import { encodeJsonRpcFrame, MessageType } from './protocol' +import { PtyHandler } from './pty-handler' +import { RelayPtySourcePublication } from './relay-pty-source-publication' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +const { mockPtySpawn } = vi.hoisted(() => ({ mockPtySpawn: vi.fn() })) + +vi.mock('node-pty', () => ({ spawn: mockPtySpawn })) + +const endpointIdentity: RelayClientSessionIdentity = { + principal: 'endpoint-principal', + authenticated: true, + allowSessionOwner: true, + authenticationKind: 'endpoint-credential' +} + +type Notification = { method: string; params: Record } + +function requestFrame(id: number, method: string, params: Record): Buffer { + return encodeJsonRpcFrame({ jsonrpc: '2.0', id, method, params }, id, 0) +} + +function notification(buffer: Buffer): Notification | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + const message = JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) + return typeof message.method === 'string' && message.id === undefined ? message : null +} + +function responseResult(buffer: Buffer, id: number): Record | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + const message = JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) + return message.id === id ? (message.result ?? null) : null +} + +describe('PtyHandler negotiated source publication', () => { + let dispatcher: RelayDispatcher + let handler: PtyHandler + let publication: RelayPtySourcePublication + let dataCallback: ((data: string) => void) | undefined + let originalPlatform: PropertyDescriptor | undefined + let writes: Buffer[] + let heldResponseId: number | null + let heldResponseSettlements: ((result: SinkWriteSettlement) => void)[] + let adapter: SshPtyConsumerSessionAdapter + let pausePty: ReturnType + + beforeEach(async () => { + vi.useFakeTimers() + originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + writes = [] + heldResponseId = null + heldResponseSettlements = [] + dataCallback = undefined + pausePty = vi.fn() + mockPtySpawn.mockReset() + mockPtySpawn.mockReturnValue({ + pid: process.pid, + onData: vi.fn((callback: (data: string) => void) => { + dataCallback = callback + }), + onExit: vi.fn(), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(), + clear: vi.fn(), + pause: pausePty, + resume: vi.fn() + }) + dispatcher = new RelayDispatcher( + (data, settle) => { + writes.push(Buffer.from(data)) + if (heldResponseId !== null && responseResult(data, heldResponseId)) { + heldResponseSettlements.push(settle) + return true + } + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + handler = new PtyHandler(dispatcher) + adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, (id) => + handler.handleSourcePublicationCapacity(id) + ) + handler.setSourcePublication(publication) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 1024 } } + }) + ) + await vi.advanceTimersByTimeAsync(0) + }) + + afterEach(async () => { + await handler.dispose({ waitForPhysicalExit: false }).catch(() => {}) + dispatcher.dispose() + if (originalPlatform) { + Object.defineProperty(process, 'platform', originalPlatform) + } + vi.useRealTimers() + }) + + async function spawn(params: Record): Promise { + dispatcher.feed(requestFrame(2, 'pty.spawn', params)) + await vi.advanceTimersByTimeAsync(0) + expect(dataCallback).toBeTypeOf('function') + } + + function sourceDataFrames(): Notification[] { + return writes + .map(notification) + .filter((frame): frame is Notification => frame?.method === 'pty.data') + } + + it('fences the first source frame behind immutable spawn and attach activation metadata', async () => { + heldResponseId = 2 + await spawn({}) + const spawnResult = writes.map((buffer) => responseResult(buffer, 2)).find(Boolean)! + const sourceActivation = spawnResult.sourceActivation as Record + + expect(sourceActivation).toMatchObject({ + status: 'pending', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: spawnResult.incarnationId, + deliveryToken: expect.any(String), + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + }) + dataCallback!('prompt') + await vi.advanceTimersByTimeAsync(8) + expect(sourceDataFrames()).toHaveLength(0) + + heldResponseSettlements[0]({ ok: true }) + const firstSource = sourceDataFrames()[0] + expect(firstSource.params).toMatchObject({ + id: spawnResult.id, + data: 'prompt', + clientGeneration: sourceActivation.clientGeneration, + ownerGeneration: sourceActivation.ownerGeneration, + ptyIncarnation: sourceActivation.ptyIncarnation, + deliveryToken: sourceActivation.deliveryToken, + sourceEndSu: 6 + }) + + dispatcher.feed(requestFrame(3, 'pty.attach', { id: spawnResult.id })) + await vi.advanceTimersByTimeAsync(0) + const attachResult = writes.map((buffer) => responseResult(buffer, 3)).find(Boolean)! + expect(attachResult.sourceActivation).toEqual(sourceActivation) + }) + + it('settles a consumed POSIX startup query before publishing its prompt', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) + await spawn({ + startupIngressVersion: PTY_STARTUP_INGRESS_VERSION, + startupIngress: { + colors: { foreground: '#2e3434', background: '#ffffff' }, + deadlineMs: 5_000 + } + }) + const query = '\x1b]10;?\x07' + + dataCallback!(query) + dataCallback!('prompt') + await vi.advanceTimersByTimeAsync(9) + + expect(sourceDataFrames().map((frame) => frame.params)).toEqual([ + expect.objectContaining({ + data: '', + rawLength: query.length, + transformed: true, + sourceLengthSu: query.length, + sourceEndSu: query.length + }), + expect.objectContaining({ + data: 'prompt', + rawLength: 6, + sourceLengthSu: 6, + sourceEndSu: query.length + 6 + }) + ]) + expect(publication.getDebugSnapshot()).toMatchObject({ sendCommitted: 2 }) + }) + + it('settles a suppressed ConPTY query before publishing its prompt', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + await spawn({ shellOverride: 'powershell.exe' }) + const query = '\x1b]10;?\x07' + + dataCallback!(query) + dataCallback!('PS> ') + await vi.advanceTimersByTimeAsync(9) + + expect(sourceDataFrames().map((frame) => frame.params)).toEqual([ + expect.objectContaining({ + data: '', + rawLength: query.length, + transformed: true, + sourceLengthSu: query.length, + sourceEndSu: query.length + }), + expect.objectContaining({ + data: 'PS> ', + rawLength: 4, + sourceLengthSu: 4, + sourceEndSu: query.length + 4 + }) + ]) + expect(publication.getDebugSnapshot()).toMatchObject({ sendCommitted: 2 }) + }) + + it('activates a fresh source consumer when an operation response is lost and retried', async () => { + const operationId = 'a'.repeat(43) + const grant = writes.map((buffer) => responseResult(buffer, 1)).find(Boolean)! + heldResponseId = 2 + dispatcher.feed( + requestFrame(2, 'pty.spawn', { + agentSessionCreateOperationId: operationId + }) + ) + await vi.advanceTimersByTimeAsync(0) + expect(heldResponseSettlements).toHaveLength(1) + expect( + writes.map((buffer) => responseResult(buffer, 2)).find(Boolean)?.sourceActivation + ).toEqual(expect.objectContaining({ deliveryToken: expect.any(String) })) + + dispatcher.invalidateClient() + expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 0 }) + + const replacementWrites: Buffer[] = [] + const replacementClientId = dispatcher.attachClient( + (data, settle) => { + replacementWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + replacementClientId, + requestFrame(3, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: grant.ownerGeneration, + ownerLease: grant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 1024 } } + }) + ) + await vi.advanceTimersByTimeAsync(0) + dispatcher.feedClient( + replacementClientId, + requestFrame(4, 'pty.spawn', { + agentSessionCreateOperationId: operationId + }) + ) + await vi.advanceTimersByTimeAsync(0) + + dataCallback!('prompt') + await vi.advanceTimersByTimeAsync(8) + + expect(mockPtySpawn).toHaveBeenCalledOnce() + expect( + responseResult(replacementWrites.find((buffer) => responseResult(buffer, 4))!, 4) + ).toMatchObject({ + id: 'pty-1', + incarnationId: expect.any(String), + sourceActivation: expect.objectContaining({ deliveryToken: expect.any(String) }) + }) + expect( + replacementWrites.map(notification).find((frame) => frame?.method === 'pty.data')?.params + ).toMatchObject({ + id: 'pty-1', + data: 'prompt', + sourceLengthSu: 6, + sourceEndSu: 6 + }) + expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1 }) + }) + + it('keeps the native PTY and V1 owner live when one subscriber saturates', async () => { + await spawn({}) + const detached: number[] = [] + const healthyWrites: Buffer[] = [] + dispatcher.onClientDetached((clientId) => detached.push(clientId)) + const saturatedId = dispatcher.attachClient(() => false, { + supportsWriteCallback: true, + writableLength: () => 16 * 1024, + writableHighWaterMark: () => 4 * 1024 * 1024 + }) + const healthyId = dispatcher.attachClient( + (data, settle) => { + healthyWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true } + ) + const payload = 's'.repeat(16 * 1024) + let admitted = 0 + while ( + dispatcher.tryNotifyPtyDataToClient(saturatedId, { id: 'saturated', data: payload }, () => {}) + ) { + admitted++ + } + + dataCallback!(payload) + await vi.advanceTimersByTimeAsync(8) + + expect(admitted).toBeGreaterThan(100) + expect(admitted).toBeLessThan(140) + expect(detached).toEqual([saturatedId]) + expect(detached).not.toContain(healthyId) + expect( + healthyWrites.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + expect(sourceDataFrames()).toHaveLength(1) + expect(publication.getDebugSnapshot()).toMatchObject({ sendCommitted: 1 }) + expect(pausePty).not.toHaveBeenCalled() + }) +}) diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index 3a2f1ae8ebf..ab853aed504 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -24,7 +24,9 @@ const { mockPtySpawn, mockPtyInstance } = vi.hoisted(() => ({ write: vi.fn(), resize: vi.fn(), kill: vi.fn(), - clear: vi.fn() + clear: vi.fn(), + pause: vi.fn(), + resume: vi.fn() } })) @@ -39,7 +41,8 @@ import { attachIdentityMismatches, formatNodePtyUnavailableMessage } from './pty-handler' -import type { RelayDispatcher } from './dispatcher' +import { RelayDispatcher } from './dispatcher' +import { encodeJsonRpcFrame } from './protocol' type TestRequestContext = { isStale: () => boolean @@ -127,6 +130,8 @@ describe('PtyHandler', () => { mockPtyInstance.resize.mockReset() mockPtyInstance.kill.mockReset() mockPtyInstance.clear.mockReset() + mockPtyInstance.pause.mockReset() + mockPtyInstance.resume.mockReset() mockPtySpawn.mockReturnValue({ ...mockPtyInstance }) @@ -162,6 +167,60 @@ describe('PtyHandler', () => { expect(notifMethods).toContain('pty.ackData') }) + it('pauses native output at the producer hard water and resumes after retained writes settle', async () => { + let onData: ((data: string) => void) | undefined + const pause = vi.fn() + const resume = vi.fn() + mockPtySpawn.mockReturnValueOnce({ + ...mockPtyInstance, + pause, + resume, + onData: vi.fn((callback: (data: string) => void) => { + onData = callback + }) + }) + const writeCallbacks: (() => void)[] = [] + let writableLength = 0 + const boundedDispatcher = new RelayDispatcher( + (data, settle) => { + writableLength += data.length + writeCallbacks.push(() => { + writableLength -= data.length + settle({ ok: true }) + }) + return true + }, + { + supportsWriteCallback: true, + writableLength: () => writableLength, + writableHighWaterMark: () => 4 * 1024 * 1024 + } + ) + const boundedHandler = new PtyHandler(boundedDispatcher) + try { + boundedDispatcher.feed( + encodeJsonRpcFrame({ jsonrpc: '2.0', id: 1, method: 'pty.spawn', params: {} }, 1, 0) + ) + await vi.advanceTimersByTimeAsync(0) + expect(onData).toBeTypeOf('function') + + onData?.('x'.repeat(1536 * 1024)) + expect(pause).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(300) + + expect(writeCallbacks.length).toBeGreaterThan(50) + expect(resume).not.toHaveBeenCalled() + for (const settle of writeCallbacks.splice(0)) { + settle() + } + await vi.advanceTimersByTimeAsync(0) + expect(resume).toHaveBeenCalledTimes(1) + } finally { + await boundedHandler.dispose({ waitForPhysicalExit: false }).catch(() => {}) + boundedDispatcher.dispose() + } + }) + it('rejects strict process inspection for a missing relay PTY', async () => { await expect(dispatcher.callRequest('pty.inspectProcess', { id: 'missing' })).rejects.toThrow( 'terminal_gone' @@ -1229,6 +1288,58 @@ describe('PtyHandler', () => { ).resolves.toEqual({ replay: 'prompt', incarnationId: expect.any(String) }) }) + it('does not carry transformed raw length into the next plain pending entry', async () => { + await handler.dispose({ waitForPhysicalExit: false }) + const admitted: Record[] = [] + let hasCapacity = false + const tryNotifyPtyData = vi.fn((params: Record) => { + if (hasCapacity) { + admitted.push(params) + } + return hasCapacity + }) + Object.assign(dispatcher, { + onLegacyPtyCapacity: vi.fn(() => vi.fn()), + tryNotifyPtyData, + tryNotifyPtyExit: vi.fn(() => true), + legacyRetentionBelowLowWater: true + }) + handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) + let dataCallback: ((data: string) => void) | undefined + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + onData: vi.fn((callback: (data: string) => void) => { + dataCallback = callback + }), + onExit: vi.fn() + }) + await dispatcher.callRequest('pty.spawn', { + startupIngressVersion: PTY_STARTUP_INGRESS_VERSION, + startupIngress: { + colors: { foreground: '#2e3434', background: '#ffffff' }, + deadlineMs: 5_000 + } + }) + + const query = '\x1b]10;?\x07' + dataCallback?.(query) + dataCallback?.('fresh') + hasCapacity = true + await vi.runAllTimersAsync() + + expect(tryNotifyPtyData).toHaveBeenCalledTimes(3) + expect(admitted).toEqual([ + { + id: 'pty-1', + data: '', + rawLength: query.length, + seq: query.length, + transformed: true + }, + { id: 'pty-1', data: 'fresh' } + ]) + }) + it('leaves startup queries untouched for an unsupported relay capability version', async () => { const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') Object.defineProperty(process, 'platform', { configurable: true, value: 'linux' }) @@ -1456,6 +1567,78 @@ describe('PtyHandler', () => { expect(dispatcher.notify).not.toHaveBeenCalledWith('pty.data', expect.anything()) }) + it('suppresses legacy replay after the V1 owner is already active', async () => { + let dataCallback: ((data: string) => void) | undefined + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + onData: vi.fn((cb: (data: string) => void) => { + dataCallback = cb + }), + onExit: vi.fn() + }) + const spawn = await spawnPty() + dataCallback!('buffered output') + handler.setSourcePublication({ + activate: vi.fn(() => 'existing'), + accepts: vi.fn(() => true), + publish: vi.fn(() => true), + dispose: vi.fn() + } as never) + + const result = await attachPty({ + id: 'pty-1', + suppressReplayNotification: true + }) + + expect(result).toEqual({ incarnationId: spawn.incarnationId }) + }) + + it('requires restore when the V1 pending-send recovery fence expires', async () => { + const spawn = await spawnPty() + const waitForPendingSend = vi.fn().mockResolvedValue(false) + const activate = vi + .fn() + .mockReturnValue({ status: 'restoreRequired', reason: 'checkpointUnavailable' }) + handler.setSourcePublication({ + activate, + accepts: vi.fn(() => true), + waitForPendingSend, + dispose: vi.fn() + } as never) + + const result = await dispatcher.callRequest( + 'pty.attach', + { + id: spawn.id, + sourceRecovery: { + status: 'checkpoint', + deliveryToken: 'old-token', + ptyIncarnation: spawn.incarnationId, + clientGeneration: 1, + ownerGeneration: 1, + acceptedSourceEndSu: 0 + } + }, + { + clientId: 2, + isStale: () => false, + onResponseSettled: vi.fn() + } as never + ) + + expect(waitForPendingSend).toHaveBeenCalledWith(spawn.id) + expect(activate).toHaveBeenCalledWith( + spawn.id, + spawn.incarnationId, + expect.anything(), + Object.freeze({ status: 'checkpointUnavailable' }) + ) + expect(result).toEqual({ + incarnationId: spawn.incarnationId, + sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' } + }) + }) + it('notifies replay on normal attach', async () => { let dataCallback: ((data: string) => void) | undefined mockPtySpawn.mockReturnValue({ @@ -1549,6 +1732,42 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(0) }) + it('retains PTY exit until the ordinary writer admits it', async () => { + await handler.dispose({ waitForPhysicalExit: false }) + let capacityListener: (() => void) | undefined + const tryNotifyPtyExit = vi.fn().mockReturnValueOnce(false).mockReturnValueOnce(true) + Object.assign(dispatcher, { + onLegacyPtyCapacity: vi.fn((listener: () => void) => { + capacityListener = listener + return vi.fn() + }), + tryNotifyPtyData: vi.fn(() => true), + tryNotifyPtyExit, + legacyRetentionBelowLowWater: true + }) + handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) + let exitCallback: ((info: { exitCode: number }) => void) | undefined + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + onData: vi.fn(), + onExit: vi.fn((callback: (info: { exitCode: number }) => void) => { + exitCallback = callback + }) + }) + + const spawn = await spawnPty() + exitCallback?.({ exitCode: 0 }) + expect(tryNotifyPtyExit).toHaveBeenCalledOnce() + + capacityListener?.() + expect(tryNotifyPtyExit).toHaveBeenLastCalledWith({ + id: 'pty-1', + code: 0, + incarnationId: spawn.incarnationId + }) + expect(tryNotifyPtyExit).toHaveBeenCalledTimes(2) + }) + it('flushes pending PTY output before notifying exit', async () => { let dataCallback: ((data: string) => void) | undefined let exitCallback: ((info: { exitCode: number }) => void) | undefined diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index 578858d16b8..4bd805c153d 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -53,6 +53,13 @@ import { agentSessionOwnerBindingsEqual, ClaimedAgentPtyOwnerRegistry } from '../shared/claimed-agent-pty-owner' +import type { RelayPtySourceOutput } from './relay-pty-source-output' +import type { RelayPtySourcePublication } from './relay-pty-source-publication' +import type { + PtySourceRecoveryRequest, + PtySourceRecoveryResult +} from '../shared/pty-source-recovery-contract' +import type { PtySourceReceivingActivation } from '../shared/pty-source-receiving-activation' import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION, AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION, @@ -79,6 +86,39 @@ function isMissingNodePtyNativeBinding(error: unknown): boolean { ) } +function parseSourceRecoveryRequest(value: unknown): PtySourceRecoveryRequest | undefined { + if (typeof value !== 'object' || value === null) { + return undefined + } + const input = value as Record + if (input.status === 'checkpointUnavailable') { + return Object.freeze({ status: 'checkpointUnavailable' }) + } + if ( + input.status !== 'checkpoint' || + typeof input.deliveryToken !== 'string' || + input.deliveryToken.length === 0 || + typeof input.ptyIncarnation !== 'string' || + input.ptyIncarnation.length === 0 || + !Number.isSafeInteger(input.clientGeneration) || + Number(input.clientGeneration) <= 0 || + !Number.isSafeInteger(input.ownerGeneration) || + Number(input.ownerGeneration) <= 0 || + !Number.isSafeInteger(input.acceptedSourceEndSu) || + Number(input.acceptedSourceEndSu) < 0 + ) { + return Object.freeze({ status: 'checkpointUnavailable' }) + } + return Object.freeze({ + status: 'checkpoint', + deliveryToken: input.deliveryToken, + ptyIncarnation: input.ptyIncarnation, + clientGeneration: Number(input.clientGeneration), + ownerGeneration: Number(input.ownerGeneration), + acceptedSourceEndSu: Number(input.acceptedSourceEndSu) + }) +} + type ManagedPty = { id: string incarnationId: string @@ -118,17 +158,17 @@ type RelayAgentSessionCreateResult = { incarnationId: string replay?: string agentSessionEnsure?: unknown + sourceActivation?: PtySourceReceivingActivation } const AGENT_SESSION_CREATE_OPERATION_ID_PATTERN = /^[A-Za-z0-9_-]{43}$/ const AGENT_SESSION_CREATE_OPERATION_RETENTION_MS = 24 * 60 * 60 * 1000 const AGENT_SESSION_CREATE_OPERATION_LIMIT = 4_096 -type PendingPtyOutput = { +type PendingPtyOutput = RelayPtySourceOutput & { data: string - rawLength?: number - transformed?: boolean - seq?: number + interactive?: boolean + sourceChunk?: RelayPtySourceOutput } type ManagedStartupCommand = { @@ -191,6 +231,8 @@ const PTY_OUTPUT_BATCH_INTERVAL_MS = 8 const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1 const PTY_OUTPUT_FLUSH_CHUNK_CHARS = 16 * 1024 const PTY_OUTPUT_FLUSH_MAX_WRITES = 2 +const PTY_OUTPUT_PRODUCER_HIGH_BYTES = 128 * 1024 +const PTY_OUTPUT_PRODUCER_LOW_BYTES = 64 * 1024 const INTERACTIVE_OUTPUT_WINDOW_MS = 100 const INTERACTIVE_OUTPUT_MAX_CHARS = 1024 const INTERACTIVE_REDRAW_MAX_CHARS = PTY_OUTPUT_FLUSH_CHUNK_CHARS @@ -309,7 +351,12 @@ export class PtyHandler { private graceTimeMs: number private graceTimer: ReturnType | null = null private outputFlushTimer: ReturnType | null = null - private pendingOutputByPty = new Map() + private pendingOutputByPty = new Map() + private pendingExitByPty = new Map() + private pausedOutputPtys = new Set() + private consumerPausedOutputPtys = new Set() + private removeLegacyCapacityListener: (() => void) | null = null + private sourcePublication: RelayPtySourcePublication | null = null private lastInputAtByPty = new Map() private interactiveOutputCharsByPty = new Map() private pendingSpawnCount = 0 @@ -335,6 +382,34 @@ export class PtyHandler { this.dispatcher = dispatcher this.graceTimeMs = graceTimeMs this.registerHandlers() + this.removeLegacyCapacityListener = + this.dispatcher.onLegacyPtyCapacity?.(() => this.handleLegacyCapacity()) ?? null + } + + setConsumerDeliveryPaused(id: string, paused: boolean): void { + if (paused) { + this.consumerPausedOutputPtys.add(id) + this.pausePtyOutput(id) + return + } + this.consumerPausedOutputPtys.delete(id) + this.maybeResumePtyOutput(id) + } + + setSourcePublication(publication: RelayPtySourcePublication): void { + this.sourcePublication = publication + } + + handleSourceCreditAvailable(id: string): void { + this.sourcePublication?.onCreditAvailable(id) + } + + handleSourcePublicationCapacity(id: string): void { + if (this.pendingOutputByPty.has(id)) { + this.scheduleOutputFlush(0) + } + this.maybeResumePtyOutput(id) + this.publishPendingExit(id) } private async loadPty(): Promise { @@ -578,16 +653,19 @@ export class PtyHandler { } this.clearStartupCommandTimer(managed) this.releaseRelayIngress(managed) + this.pausedOutputPtys.delete(managed.id) + this.consumerPausedOutputPtys.delete(managed.id) this.flushPtyOutput(managed.id) - this.dispatcher.notify('pty.exit', { + this.pendingExitByPty.set(managed.id, { id: managed.id, code: exitCode, incarnationId: managed.incarnationId }) + this.publishPendingExit(managed.id) this.notifyExitListener(managed) this.agentSessionOwners.release(managed.id) this.ptys.delete(managed.id) - this.clearPtyFlowState(managed.id) + this.clearPtyInputState(managed.id) // Why: release the ptmx fd on natural exit, else the master fd leaks until GC (docs/fix-pty-fd-leak.md). disposeManagedPty(managed) }) @@ -623,7 +701,7 @@ export class PtyHandler { private registerHandlers(): void { this.dispatcher.onRequest('pty.spawn', (p, context) => this.spawn(p, context)) - this.dispatcher.onRequest('pty.attach', (p) => this.attach(p)) + this.dispatcher.onRequest('pty.attach', (p, context) => this.attach(p, context)) this.dispatcher.onRequest('pty.shutdown', (p) => this.shutdown(p)) this.dispatcher.onRequest('pty.sendSignal', (p) => this.sendSignal(p)) this.dispatcher.onRequest('pty.getCwd', (p) => this.getCwd(p)) @@ -697,31 +775,63 @@ export class PtyHandler { data: string, meta: { rawLength?: number; transformed?: boolean; seq?: number } = {} ): void { - const existing = this.pendingOutputByPty.get(id) - if (meta.transformed === true) { - // Why: transformed spans lack a raw-to-clean slice mapping, so they can't be folded into the output batch. - if (existing) { - this.flushPtyOutput(id) + const queue = this.pendingOutputByPty.get(id) ?? [] + if (this.sourcePublication?.accepts(id)) { + queue.push({ data, ...meta }) + this.pendingOutputByPty.set(id, queue) + if (queue.length === 1 && this.shouldSendInteractiveOutputNow(id, data)) { + queue[0].interactive = true + if (this.flushPtyOutput(id)) { + return + } } - this.dispatcher.notify('pty.data', { id, data, ...meta }) + if (this.pendingProducerBytes(id) >= PTY_OUTPUT_PRODUCER_HIGH_BYTES) { + this.pausePtyOutput(id) + } + this.scheduleOutputFlush(PTY_OUTPUT_BATCH_INTERVAL_MS) return } - const pending: PendingPtyOutput = { data: (existing?.data ?? '') + data } - if (existing?.rawLength !== undefined || meta.rawLength !== undefined) { - pending.rawLength = - (existing?.rawLength ?? existing?.data.length ?? 0) + (meta.rawLength ?? data.length) + const existing = queue.at(-1) + if (meta.transformed === true) { + if (queue.length === 0) { + const transformed = { data, ...meta } + if (this.publishPtyOutput(id, transformed, false)) { + return + } + queue.push(transformed) + } else if (existing?.transformed) { + existing.data += data + existing.rawLength = (existing.rawLength ?? 0) + (meta.rawLength ?? data.length) + existing.seq = meta.seq + } else { + queue.push({ data, ...meta }) + } + this.pendingOutputByPty.set(id, queue) + this.pausePtyOutput(id) + return + } + const pending: PendingPtyOutput = existing && !existing.transformed ? existing : { data: '' } + const previousLength = pending.data.length + pending.data += data + if (pending.rawLength !== undefined || meta.rawLength !== undefined) { + pending.rawLength = (pending.rawLength ?? previousLength) + (meta.rawLength ?? data.length) } if (meta.seq !== undefined) { pending.seq = meta.seq } - if (this.shouldSendInteractiveOutputNow(id, pending.data)) { - this.pendingOutputByPty.delete(id) - this.clearOutputFlushTimerIfIdle() - // Why: send interactive echo immediately — batching must not add visible input delay for TUIs. - this.dispatcher.notify('pty.data', { id, ...pending }) - return + if (!existing || existing.transformed) { + queue.push(pending) + } + this.pendingOutputByPty.set(id, queue) + if (queue.length === 1 && this.shouldSendInteractiveOutputNow(id, pending.data)) { + pending.interactive = true + if (this.flushPtyOutput(id)) { + return + } + } + if (this.pendingProducerBytes(id) >= PTY_OUTPUT_PRODUCER_HIGH_BYTES) { + this.pausePtyOutput(id) } - this.pendingOutputByPty.set(id, pending) this.scheduleOutputFlush(PTY_OUTPUT_BATCH_INTERVAL_MS) } @@ -737,56 +847,107 @@ export class PtyHandler { // Why batch before the first send: a re-entrant sink must read the values a whole-map snapshot // would have frozen. Why the raw iterator: `for...of` would consume one entry past the limit. const pendingEntries = this.pendingOutputByPty[Symbol.iterator]() - const batch: [string, PendingPtyOutput][] = [] + const batch: [string, PendingPtyOutput[]][] = [] while (batch.length < PTY_OUTPUT_FLUSH_MAX_WRITES) { const next = pendingEntries.next() if (next.done === true) { break } - batch.push(next.value) + batch.push([next.value[0], next.value[1].map((pending) => ({ ...pending }))]) } - for (const [id, pending] of batch) { + let writes = 0 + for (const [id, queue] of batch) { this.pendingOutputByPty.delete(id) - const chunk = pending.transformed - ? pending.data - : pending.data.slice(0, PTY_OUTPUT_FLUSH_CHUNK_CHARS) - const remaining = pending.transformed ? '' : pending.data.slice(PTY_OUTPUT_FLUSH_CHUNK_CHARS) - if (remaining) { - this.pendingOutputByPty.set(id, { - data: remaining, - ...(pending.rawLength === undefined ? {} : { rawLength: remaining.length }), - seq: pending.seq - }) + if (this.flushPtyOutput(id, queue)) { + writes++ } - const chunkRawLength = pending.transformed - ? pending.rawLength - : pending.rawLength === undefined - ? undefined - : chunk.length - const chunkSeq = - pending.seq === undefined ? undefined : pending.seq - (pending.data.length - chunk.length) - this.dispatcher.notify('pty.data', { - id, - data: chunk, - ...(chunkSeq === undefined ? {} : { seq: chunkSeq }), - ...(chunkRawLength === undefined ? {} : { rawLength: chunkRawLength }), - ...(pending.transformed ? { transformed: true } : {}) - }) } - if (this.pendingOutputByPty.size > 0 && batch.length > 0) { + if (this.pendingOutputByPty.size > 0 && writes > 0) { // Why: yield between slices of a large chunk so client input and control frames can interleave. this.scheduleOutputFlush(PTY_OUTPUT_DRAIN_CONTINUE_MS) } } - private flushPtyOutput(id: string): void { - const pending = this.pendingOutputByPty.get(id) - if (!pending) { - return + private flushPtyOutput(id: string, capturedQueue?: PendingPtyOutput[]): boolean { + const queue = capturedQueue ?? this.pendingOutputByPty.get(id) + const pending = queue?.[0] + if (!queue || !pending) { + this.publishPendingExit(id) + return true } - this.pendingOutputByPty.delete(id) - this.dispatcher.notify('pty.data', { id, ...pending }) + const desiredChars = pending.transformed + ? pending.data.length + : Math.min(pending.data.length, PTY_OUTPUT_FLUSH_CHUNK_CHARS) + const sourceOnlyEmission = + pending.transformed === true && pending.data.length === 0 && (pending.rawLength ?? 0) > 0 + const paramsWithoutData = { + id, + ...(pending.seq === undefined ? {} : { seq: pending.seq }), + ...(pending.rawLength === undefined ? {} : { rawLength: pending.rawLength }), + ...(pending.transformed ? { transformed: true } : {}) + } + let chunkChars = pending.transformed + ? desiredChars + : (this.dispatcher.maxLegacyPtyDataChars?.(paramsWithoutData, pending.data, desiredChars) ?? + desiredChars) + if ( + chunkChars > 0 && + chunkChars < pending.data.length && + pending.data.charCodeAt(chunkChars - 1) >= 0xd800 && + pending.data.charCodeAt(chunkChars - 1) <= 0xdbff + ) { + chunkChars-- + } + if ( + (!sourceOnlyEmission && chunkChars <= 0) || + (pending.transformed && chunkChars !== pending.data.length) + ) { + this.pendingOutputByPty.set(id, queue) + this.pausePtyOutput(id) + return false + } + const chunk = pending.data.slice(0, chunkChars) + const remaining = pending.data.slice(chunkChars) + const chunkRawLength = pending.transformed + ? pending.rawLength + : pending.rawLength === undefined + ? undefined + : chunk.length + const chunkSeq = + pending.seq === undefined ? undefined : pending.seq - (pending.data.length - chunk.length) + const sourceChunk = + pending.sourceChunk ?? + ({ + data: chunk, + ...(chunkSeq === undefined ? {} : { seq: chunkSeq }), + ...(chunkRawLength === undefined ? {} : { rawLength: chunkRawLength }), + ...(pending.transformed ? { transformed: true } : {}) + } satisfies RelayPtySourceOutput) + pending.sourceChunk = sourceChunk + const published = this.publishPtyOutput(id, sourceChunk, pending.interactive === true) + if (!published) { + this.pendingOutputByPty.set(id, queue) + this.pausePtyOutput(id) + return false + } + if (remaining) { + queue[0] = { + data: remaining, + ...(pending.rawLength === undefined ? {} : { rawLength: remaining.length }), + seq: pending.seq + } + } else { + queue.shift() + } + if (queue.length === 0) { + this.pendingOutputByPty.delete(id) + this.publishPendingExit(id) + } else { + this.pendingOutputByPty.set(id, queue) + } + this.maybeResumePtyOutput(id) this.clearOutputFlushTimerIfIdle() + return true } private clearOutputFlushTimerIfIdle(): void { @@ -799,9 +960,121 @@ export class PtyHandler { private clearPtyFlowState(id: string): void { this.pendingOutputByPty.delete(id) + this.pendingExitByPty.delete(id) + this.pausedOutputPtys.delete(id) + this.consumerPausedOutputPtys.delete(id) + this.clearPtyInputState(id) + this.clearOutputFlushTimerIfIdle() + } + + private clearPtyInputState(id: string): void { this.lastInputAtByPty.delete(id) this.interactiveOutputCharsByPty.delete(id) - this.clearOutputFlushTimerIfIdle() + } + + private publishPtyOutput( + id: string, + output: RelayPtySourceOutput, + interactive: boolean + ): boolean { + if (this.sourcePublication?.accepts(id)) { + return this.sourcePublication.publish(id, output, interactive) + } + if (this.dispatcher.tryNotifyPtyData) { + return this.dispatcher.tryNotifyPtyData( + { + id, + data: output.data, + ...(output.seq === undefined ? {} : { seq: output.seq }), + ...(output.rawLength === undefined ? {} : { rawLength: output.rawLength }), + ...(output.transformed ? { transformed: true } : {}) + }, + { interactive } + ) + } + this.dispatcher.notify('pty.data', { + id, + data: output.data, + ...(output.seq === undefined ? {} : { seq: output.seq }), + ...(output.rawLength === undefined ? {} : { rawLength: output.rawLength }), + ...(output.transformed ? { transformed: true } : {}) + }) + return true + } + + private publishPendingExit(id: string): void { + if (this.pendingOutputByPty.has(id)) { + return + } + const exit = this.pendingExitByPty.get(id) + if (!exit) { + return + } + if (this.sourcePublication?.accepts(id)) { + if ( + !this.sourcePublication.sealAndPublishExit(exit) || + !this.sourcePublication.exitPublicationSettled(id) + ) { + return + } + this.pendingExitByPty.delete(id) + return + } + const published = this.dispatcher.tryNotifyPtyExit + ? this.dispatcher.tryNotifyPtyExit(exit) + : (this.dispatcher.notify('pty.exit', exit), true) + if (!published) { + return + } + this.pendingExitByPty.delete(id) + } + + private pendingProducerBytes(id: string): number { + return (this.pendingOutputByPty.get(id) ?? []).reduce( + (total, pending) => + total + Math.max(Buffer.byteLength(pending.data, 'utf8'), 2 * pending.data.length) + 128, + 0 + ) + } + + private pausePtyOutput(id: string): void { + if (this.pausedOutputPtys.has(id)) { + return + } + const managed = this.ptys.get(id) + if (!managed || managed.disposed) { + return + } + this.pausedOutputPtys.add(id) + managed.pty.pause() + } + + private maybeResumePtyOutput(id: string): void { + if ( + !this.pausedOutputPtys.has(id) || + this.consumerPausedOutputPtys.has(id) || + this.pendingProducerBytes(id) > PTY_OUTPUT_PRODUCER_LOW_BYTES || + this.dispatcher.legacyRetentionBelowLowWater === false + ) { + return + } + const managed = this.ptys.get(id) + this.pausedOutputPtys.delete(id) + if (managed && !managed.disposed) { + managed.pty.resume() + } + } + + private handleLegacyCapacity(): void { + if (this.pendingOutputByPty.size > 0) { + this.scheduleOutputFlush(0) + } + for (const id of Array.from(this.pendingExitByPty.keys())) { + this.publishPendingExit(id) + } + for (const id of Array.from(this.pausedOutputPtys)) { + this.maybeResumePtyOutput(id) + } } private beginPtyCreation(operationPaths: readonly (string | undefined)[]): () => void { @@ -874,7 +1147,12 @@ export class PtyHandler { } const existing = this.agentSessionCreateOperations.get(operationId) if (existing) { - return await existing + const result = await existing + this.sourcePublication?.activate(result.id, result.incarnationId, context) + const sourceActivation = + context && this.sourcePublication?.receivingActivation?.(result.id, context.clientId) + const { sourceActivation: _staleActivation, ...stableResult } = result + return { ...stableResult, ...(sourceActivation ? { sourceActivation } : {}) } } if (this.agentSessionCreateOperations.size >= AGENT_SESSION_CREATE_OPERATION_LIMIT) { throw new Error('agent_session_operation_capacity') @@ -981,10 +1259,14 @@ export class PtyHandler { } managed.agentSessionOwners = this.agentSessionOwners.listForPty(managed.id) const adoptedReplay = result.disposition === 'adopted' ? managed.buffered.read() : '' + this.sourcePublication?.activate(managed.id, managed.incarnationId, context) + const sourceActivation = + context && this.sourcePublication?.receivingActivation?.(managed.id, context.clientId) return { id: managed.id, incarnationId: managed.incarnationId, agentSessionEnsure: result, + ...(sourceActivation ? { sourceActivation } : {}), ...(adoptedReplay ? { replay: adoptedReplay } : {}) } } catch (error) { @@ -1004,7 +1286,11 @@ export class PtyHandler { params: Record, context?: RequestContext, onPhysicalSpawnCommitted?: () => void - ): Promise<{ id: string; incarnationId: string }> { + ): Promise<{ + id: string + incarnationId: string + sourceActivation?: PtySourceReceivingActivation + }> { const pty = await this.loadPty() if (!pty) { throw new Error(formatNodePtyUnavailableMessage(process.platform)) @@ -1143,6 +1429,9 @@ export class PtyHandler { } : {}) } + this.sourcePublication?.activate(id, managed.incarnationId, context) + const sourceActivation = + context && this.sourcePublication?.receivingActivation?.(id, context.clientId) this.wireAndStore(managed) if (context?.isStale() && !params.agentSessionEnsure && !params.agentSessionCreateOperationId) { // Why: if the client reconnected while pty.spawn was in flight, the @@ -1158,12 +1447,22 @@ export class PtyHandler { : STARTUP_COMMAND_WRITE_DELAY_MS ) } - return { id, incarnationId: managed.incarnationId } + return { + id, + incarnationId: managed.incarnationId, + ...(sourceActivation ? { sourceActivation } : {}) + } } private async attach( - params: Record - ): Promise<{ incarnationId: string; replay?: string }> { + params: Record, + context?: RequestContext + ): Promise<{ + incarnationId: string + replay?: string + sourceRecovery?: PtySourceRecoveryResult + sourceActivation?: PtySourceReceivingActivation + }> { const id = params.id as string const managed = this.ptys.get(id) // Why: after dispose, pty.kill is a POSIX no-op; treat disposed as not-found so failures aren't silent. @@ -1197,6 +1496,35 @@ export class PtyHandler { } managed.startupIngress?.snapshotBarrier() + let sourceRecovery = parseSourceRecoveryRequest(params.sourceRecovery) + if ( + sourceRecovery?.status === 'checkpoint' && + this.sourcePublication && + !(await this.sourcePublication.waitForPendingSend(id)) + ) { + sourceRecovery = Object.freeze({ status: 'checkpointUnavailable' }) + } + const activation = this.sourcePublication?.activate( + id, + managed.incarnationId, + context, + sourceRecovery + ) + const sourceActivation = + context && this.sourcePublication?.receivingActivation?.(id, context.clientId) + if (typeof activation === 'object') { + return { + incarnationId: managed.incarnationId, + sourceRecovery: activation, + ...(sourceActivation ? { sourceActivation } : {}) + } + } + if (activation === 'existing' && this.sourcePublication?.accepts(id)) { + return { + incarnationId: managed.incarnationId, + ...(sourceActivation ? { sourceActivation } : {}) + } + } // Why: renderer hasn't registered replay handlers yet during spawn, so return to the caller instead of notifying too early. // Why: buffer intentionally NOT cleared after replay (client clears xterm first) so later restarts still replay full history. @@ -1205,12 +1533,20 @@ export class PtyHandler { // Why: drop pending batched bytes already in the replay buffer so attach doesn't render them twice. this.pendingOutputByPty.delete(id) this.clearOutputFlushTimerIfIdle() + this.maybeResumePtyOutput(id) if (params.suppressReplayNotification) { - return { incarnationId: managed.incarnationId, replay } + return { + incarnationId: managed.incarnationId, + replay, + ...(sourceActivation ? { sourceActivation } : {}) + } } this.dispatcher.notify('pty.replay', { id, data: replay }) } - return { incarnationId: managed.incarnationId } + return { + incarnationId: managed.incarnationId, + ...(sourceActivation ? { sourceActivation } : {}) + } } private writeData(params: Record): void { @@ -1603,6 +1939,8 @@ export class PtyHandler { if (this.disposePromise) { return this.disposePromise } + this.removeLegacyCapacityListener?.() + this.removeLegacyCapacityListener = null this.agentSessionCreateOperations.clear() const disposePromise = this.disposePtys(options.waitForPhysicalExit !== false) this.disposePromise = disposePromise @@ -1627,8 +1965,13 @@ export class PtyHandler { this.outputFlushTimer = null } this.pendingOutputByPty.clear() + this.pendingExitByPty.clear() + this.pausedOutputPtys.clear() + this.consumerPausedOutputPtys.clear() this.lastInputAtByPty.clear() this.interactiveOutputCharsByPty.clear() + this.sourcePublication?.dispose() + this.sourcePublication = null const results = await Promise.allSettled( [...this.ptys.values()].map((managed) => this.disposePtyForRelayShutdown(managed, waitForPhysicalExit) diff --git a/src/relay/pty-source-cancellation-index.ts b/src/relay/pty-source-cancellation-index.ts new file mode 100644 index 00000000000..740b7fba50c --- /dev/null +++ b/src/relay/pty-source-cancellation-index.ts @@ -0,0 +1,48 @@ +import type { PtySourceDeliveryCancellation } from '../shared/pty-source-credit-contract' +import type { PtyConsumerSessionGrant } from '../shared/pty-consumer-session' + +export const RECENT_PTY_SOURCE_CANCELLATION_LIMIT = 256 + +export class RecentPtySourceCancellationIndex { + private readonly proofs = new Map() + + remember(proof: PtySourceDeliveryCancellation): void { + this.proofs.set(proof.deliveryToken, proof) + while (this.proofs.size > RECENT_PTY_SOURCE_CANCELLATION_LIMIT) { + this.proofs.delete(this.proofs.keys().next().value!) + } + } + + get(deliveryToken: string): PtySourceDeliveryCancellation | undefined { + return this.proofs.get(deliveryToken) + } + + owned( + deliveryToken: string, + params: Record, + grant: Readonly | null + ): PtySourceDeliveryCancellation | undefined { + const proof = this.proofs.get(deliveryToken) + return grant?.capabilities?.outputFlowControl && + proof?.clientGeneration === grant.clientGeneration && + params.id === proof.id && + Number(params.clientGeneration) === proof.clientGeneration && + Number(params.ownerGeneration) === proof.ownerGeneration + ? proof + : undefined + } + + clear(): void { + this.proofs.clear() + } +} + +export function ptySourceCancellationResult( + proof: Readonly<{ sentEndSu: number; creditedEndSu: number }> +) { + return Object.freeze({ + canceled: true as const, + sentEndSu: proof.sentEndSu, + creditedEndSu: proof.creditedEndSu + }) +} diff --git a/src/relay/pty-source-credit-ledger.test.ts b/src/relay/pty-source-credit-ledger.test.ts new file mode 100644 index 00000000000..e34ff5b10e0 --- /dev/null +++ b/src/relay/pty-source-credit-ledger.test.ts @@ -0,0 +1,512 @@ +import { describe, expect, it } from 'vitest' +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import { RelayPtySourceCreditLedger } from './pty-source-credit-ledger' + +function identity( + deliveryToken = 'token-1', + overrides: Partial = {} +): PtySourceDeliveryIdentity { + return { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 2, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken, + ...overrides + } +} + +function append( + ledger: RelayPtySourceCreditLedger, + owner: PtySourceDeliveryIdentity, + data: string, + spanId = `span-${data}` +): void { + const start = ledger.snapshot(owner).receivedEndSu + ledger.append(owner, { + spanId, + data, + displayStart: start, + displayEnd: start + data.length, + splittable: true, + transform: { + transformed: false, + rawLengthSu: data.length, + scalarSafe: true + } + }) +} + +function drainOne( + ledger: RelayPtySourceCreditLedger, + owner: PtySourceDeliveryIdentity, + maxSourceSu = 16 * 1024 +) { + const reservation = ledger.reserveNextSend(owner, maxSourceSu) + if (reservation) { + ledger.commitSend(reservation) + } + return reservation +} + +describe('RelayPtySourceCreditLedger', () => { + it('never exceeds a token source window across generated send/ACK sequences', () => { + for (let seed = 1; seed <= 40; seed++) { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity(`token-${seed}`) + const windowSu = 7 + (seed % 17) + ledger.open(owner, windowSu) + append(ledger, owner, 'x'.repeat(100), `span-${seed}`) + + for (let turn = 0; turn < 100; turn++) { + const reservation = drainOne(ledger, owner, 1 + ((seed * 13 + turn * 7) % 19)) + const snapshot = ledger.snapshot(owner) + expect(snapshot.sentEndSu - snapshot.creditedEndSu).toBeLessThanOrEqual(windowSu) + if (snapshot.sentEndSu > snapshot.creditedEndSu && (turn + seed) % 3 === 0) { + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: owner.clientGeneration, + ownerGeneration: owner.ownerGeneration, + deliveryToken: owner.deliveryToken, + creditedEndSu: snapshot.sentEndSu + }) + } + if (!reservation && snapshot.creditedEndSu === 100) { + break + } + } + } + }) + + it('slices splittable source without splitting a surrogate pair', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 3) + append(ledger, owner, `a😀b`) + + expect(drainOne(ledger, owner, 2)?.span.data).toBe('a') + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: owner.deliveryToken, + creditedEndSu: 1 + }) + expect(drainOne(ledger, owner, 2)?.span.data).toBe('😀') + }) + + it('never emits the leading half of a surrogate pair', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 1) + append(ledger, owner, `😀`) + + expect(() => drainOne(ledger, owner, 1)).toThrow('surrogate pair') + expect(ledger.snapshot(owner)).toMatchObject({ sentEndSu: 0, creditedEndSu: 0 }) + }) + + it('holds an indivisible transform that does not fit the remaining window', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 4, 2) + ledger.append(owner, { + spanId: 'transform', + data: 'Z', + displayStart: 0, + displayEnd: 1, + splittable: false, + transform: { transformed: true, rawLengthSu: 5, scalarSafe: true } + }) + + expect(ledger.reserveNextSend(owner)).toBeNull() + expect(ledger.snapshot(owner)).toMatchObject({ sentEndSu: 2, creditedEndSu: 2 }) + }) + + it('rejects wrong owners, stale tokens, and over-credit without clamping', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 16) + append(ledger, owner, 'abcd') + drainOne(ledger, owner) + + expect(() => + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: 99, + ownerGeneration: owner.ownerGeneration, + deliveryToken: owner.deliveryToken, + creditedEndSu: 1 + }) + ).toThrow('does not own') + expect(() => + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: owner.clientGeneration, + ownerGeneration: owner.ownerGeneration, + deliveryToken: owner.deliveryToken, + creditedEndSu: 5 + }) + ).toThrow('exceeds sent') + expect(() => ledger.snapshot(identity('stale'))).toThrow('stale') + }) + + it('rejects cumulative ACKs inside a committed frame boundary', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 16) + append(ledger, owner, 'abcd') + drainOne(ledger, owner) + + expect(() => + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: owner.deliveryToken, + creditedEndSu: 2 + }) + ).toThrow('boundary') + expect(ledger.snapshot(owner).creditedEndSu).toBe(0) + }) + + it('reserves an exact early ACK until the matching send settles', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 4) + append(ledger, owner, 'data') + const pending = ledger.reserveNextSend(owner)! + + expect( + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: owner.clientGeneration, + ownerGeneration: owner.ownerGeneration, + deliveryToken: owner.deliveryToken, + creditedEndSu: 4 + }) + ).toBe('reserved') + expect(ledger.snapshot(owner)).toMatchObject({ sentEndSu: 0, creditedEndSu: 0 }) + + ledger.commitSend(pending) + expect(ledger.snapshot(owner)).toMatchObject({ sentEndSu: 4, creditedEndSu: 4 }) + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 0, dataBytes: 0, spans: 0 }) + }) + + it('retains reserved credit across an exact same-token send retry', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 4) + append(ledger, owner, 'data') + const failed = ledger.reserveNextSend(owner)! + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: owner.clientGeneration, + ownerGeneration: owner.ownerGeneration, + deliveryToken: owner.deliveryToken, + creditedEndSu: 4 + }) + + ledger.rollbackSend(failed) + expect( + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: owner.clientGeneration, + ownerGeneration: owner.ownerGeneration, + deliveryToken: owner.deliveryToken, + creditedEndSu: 4 + }) + ).toBe('reserved') + expect(ledger.reserveNextSend(owner, 3)).toBeNull() + const retry = ledger.reserveNextSend(owner)! + ledger.commitSend(retry) + + expect(ledger.snapshot(owner)).toMatchObject({ sentEndSu: 4, creditedEndSu: 4 }) + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 0, dataBytes: 0, spans: 0 }) + }) + + it('keeps sealed-unsettled state until exit and suffix ACK are both published', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 8) + append(ledger, owner, 'tail') + drainOne(ledger, owner) + ledger.seal(owner) + ledger.settleExitPublication(owner, { ok: true }) + + expect(ledger.snapshot(owner).state).toBe('sealed-unsettled') + ledger.acknowledge(owner, { + id: owner.id, + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: owner.deliveryToken, + creditedEndSu: 4 + }) + expect(ledger.snapshot(owner).state).toBe('closed') + }) + + it('cannot publish exit while a preceding source span is unsent or reserved', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 8) + append(ledger, owner, 'tail') + ledger.seal(owner) + + expect(() => ledger.settleExitPublication(owner, { ok: true })).toThrow('preceding') + }) + + it('rotates tokens with exact recovery and rejects the stale delivery', () => { + const ledger = new RelayPtySourceCreditLedger() + const oldOwner = identity() + const replacement = identity('token-2', { + clientGeneration: 4, + ownerGeneration: 5 + }) + ledger.open(oldOwner, 16) + append(ledger, oldOwner, 'abcdefgh') + drainOne(ledger, oldOwner, 3) + ledger.acknowledge(oldOwner, { + id: oldOwner.id, + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: oldOwner.deliveryToken, + creditedEndSu: 3 + }) + drainOne(ledger, oldOwner, 2) + + const rotation = ledger.rotate(oldOwner, replacement, 5, 16) + expect(rotation.recovery.map((span) => span.data).join('')).toBe('fgh') + expect(rotation.cancellation).toMatchObject({ + remainingStartSu: 3, + remainingEndSu: 5, + replacementDeliveryToken: 'token-2' + }) + expect(() => + ledger.acknowledge(oldOwner, { + id: oldOwner.id, + clientGeneration: 2, + ownerGeneration: 3, + deliveryToken: oldOwner.deliveryToken, + creditedEndSu: 8 + }) + ).toThrow() + expect(() => + ledger.open(identity('token-3', { clientGeneration: 6, ownerGeneration: 7 }), 16) + ).toThrow('upstream owner') + }) + + it('rejects non-boundary and pending-send recovery with zero mutation', () => { + const ledger = new RelayPtySourceCreditLedger() + const oldOwner = identity() + const replacement = identity('token-2', { + clientGeneration: 4, + ownerGeneration: 5 + }) + ledger.open(oldOwner, 16) + append(ledger, oldOwner, 'abcdefgh') + drainOne(ledger, oldOwner, 4) + const before = ledger.snapshot(oldOwner) + + expect(() => ledger.rotate(oldOwner, replacement, 3, 16)).toThrow('checkpoint') + expect(ledger.snapshot(oldOwner)).toEqual(before) + + const pending = ledger.reserveNextSend(oldOwner, 2)! + expect(() => ledger.rotate(oldOwner, replacement, 4, 16)).toThrow('checkpoint') + expect(ledger.snapshot(oldOwner)).toEqual(before) + ledger.rollbackSend(pending) + expect(ledger.snapshot(oldOwner)).toEqual(before) + }) + + it('rejects a recovery checkpoint beyond source accepted by the sink', () => { + const ledger = new RelayPtySourceCreditLedger() + const oldOwner = identity() + const replacement = identity('token-2', { + clientGeneration: 4, + ownerGeneration: 5 + }) + ledger.open(oldOwner, 16) + append(ledger, oldOwner, 'abcdefgh') + drainOne(ledger, oldOwner, 4) + + expect(() => ledger.rotate(oldOwner, replacement, 8, 16)).toThrow('checkpoint') + expect(ledger.snapshot(oldOwner)).toMatchObject({ + state: 'active', + receivedEndSu: 8, + sentEndSu: 4 + }) + }) + + it('accepts an exact receiver checkpoint for a rolled-back send attempt', () => { + const ledger = new RelayPtySourceCreditLedger() + const oldOwner = identity() + const replacement = identity('token-2', { + clientGeneration: 4, + ownerGeneration: 5 + }) + ledger.open(oldOwner, 16) + append(ledger, oldOwner, 'abcdefgh') + const attempted = ledger.reserveNextSend(oldOwner, 4)! + ledger.rollbackSend(attempted) + + const rotation = ledger.rotate(oldOwner, replacement, 4, 16) + + expect(rotation.recovery.map((span) => span.data).join('')).toBe('efgh') + expect(rotation.cancellation).toMatchObject({ sentEndSu: 4, creditedEndSu: 0 }) + expect(ledger.snapshot(replacement)).toMatchObject({ + receivedEndSu: 8, + sentEndSu: 4, + creditedEndSu: 4 + }) + }) + + it('generation-closes every retained token exactly once', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 8) + append(ledger, owner, 'data') + + expect(ledger.closeGeneration(1)).toBe(1) + expect(ledger.closeGeneration(1)).toBe(0) + expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', generationClosed: true }) + expect(ledger.retainedSourceSu()).toBe(0) + }) + + it('enforces an aggregate retained-source budget across PTYs', () => { + const ledger = new RelayPtySourceCreditLedger({ + maxRetainedSourceSu: 8, + maxAggregateRetainedSourceSu: 6 + }) + const first = identity('token-a') + const second = identity('token-b', { + id: 'pty-2', + ptyIncarnation: 'incarnation-2' + }) + ledger.open(first, 8) + ledger.open(second, 8) + append(ledger, first, 'abcd') + + expect(() => append(ledger, second, 'xyz')).toThrow('Aggregate') + expect(ledger.retainedSourceSu()).toBe(4) + }) + + it('bounds exact UTF-8 retained bytes for transformed spans', () => { + const ledger = new RelayPtySourceCreditLedger({ + maxRetainedDataBytes: 7, + maxAggregateRetainedDataBytes: 7 + }) + const owner = identity() + ledger.open(owner, 8) + + expect(() => + ledger.append(owner, { + spanId: 'expanded', + data: '😀😀', + displayStart: 0, + displayEnd: 4, + splittable: false, + transform: { transformed: true, rawLengthSu: 1, scalarSafe: false } + }) + ).toThrow('encoded-data') + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 0, dataBytes: 0, spans: 0 }) + }) + + it('charges UTF-16 storage and one record per retained source frame', () => { + const ledger = new RelayPtySourceCreditLedger({ + maxRetainedDataBytes: 391, + maxAggregateRetainedDataBytes: 391 + }) + const owner = identity() + ledger.open(owner, 8) + append(ledger, owner, '\u0000'.repeat(64), 'utf16-heavy') + + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 64, dataBytes: 256, spans: 1 }) + expect(() => append(ledger, owner, 'a'.repeat(8), 'fragment')).toThrow('retained encoded-data') + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 64, dataBytes: 256, spans: 1 }) + }) + + it('enforces per-PTY and aggregate source, charged-byte, and frame caps across fragments', () => { + const limits = { + maxRetainedSourceSu: 6, + maxAggregateRetainedSourceSu: 9, + maxRetainedDataBytes: 500, + maxAggregateRetainedDataBytes: 700, + maxRetainedSpans: 2, + maxAggregateRetainedSpans: 3 + } + const first = identity('token-a') + const second = identity('token-b', { + id: 'pty-2', + ptyIncarnation: 'incarnation-2' + }) + const ledger = new RelayPtySourceCreditLedger(limits) + ledger.open(first, 8) + ledger.open(second, 8) + append(ledger, first, 'aa', 'first-a') + append(ledger, first, 'bb', 'first-b') + append(ledger, second, 'cc', 'second-a') + + expect(ledger.retentionSnapshot()).toEqual({ + sourceSu: 6, + dataBytes: 396, + spans: 3 + }) + expect(() => append(ledger, first, 'c', 'first-frame-cap')).toThrow('retained-span') + expect(() => append(ledger, second, 'dddd', 'aggregate-source-cap')).toThrow('Aggregate') + + const byteLimited = new RelayPtySourceCreditLedger({ + ...limits, + maxRetainedSourceSu: 200, + maxAggregateRetainedSourceSu: 300, + maxAggregateRetainedSpans: 10, + maxRetainedSpans: 10, + maxAggregateRetainedDataBytes: 650 + }) + byteLimited.open(first, 8) + byteLimited.open(second, 8) + append(byteLimited, first, '\u0000'.repeat(64), 'first-utf16') + append(byteLimited, second, '\u0000'.repeat(64), 'second-utf16') + expect(() => append(byteLimited, second, '\u0000'.repeat(8), 'aggregate-byte-cap')).toThrow( + 'Aggregate' + ) + }) + + it('bounds retained frame metadata independently of source units', () => { + const ledger = new RelayPtySourceCreditLedger({ + maxRetainedSpans: 2, + maxAggregateRetainedSpans: 2 + }) + const owner = identity() + ledger.open(owner, 8) + append(ledger, owner, 'a', 'span-a') + append(ledger, owner, 'b', 'span-b') + + expect(() => append(ledger, owner, 'c', 'span-c')).toThrow('retained-span') + expect(ledger.retentionSnapshot()).toEqual({ sourceSu: 2, dataBytes: 260, spans: 2 }) + }) + + it('bounds closed token tombstones', () => { + const ledger = new RelayPtySourceCreditLedger() + const owners = Array.from({ length: 300 }, (_, index) => + identity(`token-${index}`, { + id: `pty-${index}`, + ptyIncarnation: `incarnation-${index}` + }) + ) + for (const owner of owners) { + ledger.open(owner, 8) + ledger.cancel(owner, 'test') + } + + expect(() => ledger.snapshot(owners[0])).toThrow('stale') + expect(ledger.snapshot(owners.at(-1)!)).toMatchObject({ state: 'closed' }) + }) + + it('rejects reopening a recently closed one-use token', () => { + const ledger = new RelayPtySourceCreditLedger() + const owner = identity() + ledger.open(owner, 8) + ledger.cancel(owner, 'test') + + expect(() => ledger.open(owner, 8)).toThrow('already used') + }) +}) diff --git a/src/relay/pty-source-credit-ledger.ts b/src/relay/pty-source-credit-ledger.ts new file mode 100644 index 00000000000..0a697ea30b0 --- /dev/null +++ b/src/relay/pty-source-credit-ledger.ts @@ -0,0 +1,323 @@ +import { + ptySourceDeliveryKey, + ptySourceSpanIsSplittable, + samePtySourceDelivery, + type PtySourceCreditAck, + type PtySourceDeliveryCancellation, + type PtySourceDeliveryIdentity, + type PtySourceDeliverySnapshot, + type PtySourceSpan +} from '../shared/pty-source-credit-contract' +import { + assertNonNegativeSafeInteger, + assertPositiveSafeInteger, + assertPtySourceIdentity +} from '../shared/pty-source-credit-validation' +import { + resolvePtySourceCreditLimits, + type PtySourceCreditLedgerOptions, + type PtySourceCreditLimits +} from './pty-source-credit-limits' +import { + CLOSED_DELIVERY_TOMBSTONE_LIMIT, + closeDeliveryGeneration, + createAppendedSourceSpan, + createDeliveryCancellation, + createDeliveryRecord, + createReplacementDeliveryRecord, + MAX_SOURCE_SPAN_DATA_BYTES, + ptyOwnerKey, + retainedDataBytesTotal, + retainedSpanTotal, + retainedSourceTotal, + sliceForSend, + snapshotDeliveryRecord, + type DeliveryRecord, + type PtySourceAppendInput, + type PtySourceSendReservation +} from './pty-source-credit-record' +import { + acknowledgeDeliveryRecord, + commitPtySourceSend, + reservedPtySourceSendLength, + rollbackPtySourceSend, + type PtySourceAckResult +} from './pty-source-credit-settlement' +import { chargedPtyRetainedStringBytes } from '../shared/pty-retained-string-memory' + +export type { PtySourceSendReservation } from './pty-source-credit-record' +export type { PtySourceCreditLedgerOptions } from './pty-source-credit-limits' + +export class RelayPtySourceCreditLedger { + private readonly deliveries = new Map() + private readonly upstreamOwnerByPty = new Map() + private readonly closedSnapshots = new Map() + private readonly limits: PtySourceCreditLimits + private nextReservationId = 1 + + constructor(options: PtySourceCreditLedgerOptions = {}) { + this.limits = resolvePtySourceCreditLimits(options) + } + + open( + identity: PtySourceDeliveryIdentity, + windowSu: number, + checkpointSourceEndSu = 0 + ): PtySourceDeliverySnapshot { + assertPtySourceIdentity(identity) + assertPositiveSafeInteger(windowSu, 'windowSu') + assertNonNegativeSafeInteger(checkpointSourceEndSu, 'checkpointSourceEndSu') + const key = ptySourceDeliveryKey(identity) + if (this.deliveries.has(key) || this.closedSnapshots.has(key)) { + throw new Error('PTY source delivery token was already used') + } + const ownerKey = ptyOwnerKey(identity) + if (this.upstreamOwnerByPty.has(ownerKey)) { + throw new Error('PTY source delivery already has an upstream owner') + } + const record = createDeliveryRecord(identity, windowSu, checkpointSourceEndSu) + this.deliveries.set(key, record) + this.upstreamOwnerByPty.set(ownerKey, key) + return snapshotDeliveryRecord(record) + } + + append(identity: PtySourceDeliveryIdentity, input: PtySourceAppendInput): PtySourceSpan { + const record = this.requireActive(identity) + const sourceEndSu = record.receivedEndSu + input.transform.rawLengthSu + const encodedDataBytes = Buffer.byteLength(input.data, 'utf8') + const retainedDataBytes = chargedPtyRetainedStringBytes(input.data) + if (encodedDataBytes > MAX_SOURCE_SPAN_DATA_BYTES) { + throw new Error('PTY source span encoded-data budget exceeded') + } + if (sourceEndSu - record.creditedEndSu > this.limits.maxRetainedSourceSu) { + throw new Error('PTY source retained-range budget exceeded') + } + if ( + this.retainedSourceSu() + input.transform.rawLengthSu > + this.limits.maxAggregateRetainedSourceSu + ) { + throw new Error('Aggregate PTY source retained-range budget exceeded') + } + if (record.retainedDataBytes + retainedDataBytes > this.limits.maxRetainedDataBytes) { + throw new Error('PTY source retained encoded-data budget exceeded') + } + if (this.retainedDataBytes() + retainedDataBytes > this.limits.maxAggregateRetainedDataBytes) { + throw new Error('Aggregate PTY source retained encoded-data budget exceeded') + } + if (record.spans.length + 1 > this.limits.maxRetainedSpans) { + throw new Error('PTY source retained-span budget exceeded') + } + if (this.retainedSpans() + 1 > this.limits.maxAggregateRetainedSpans) { + throw new Error('Aggregate PTY source retained-span budget exceeded') + } + const span = createAppendedSourceSpan(record, input) + record.spans.push(span) + record.retainedDataBytes += retainedDataBytes + record.receivedEndSu = sourceEndSu + return span + } + + reserveNextSend( + identity: PtySourceDeliveryIdentity, + maxSourceSu = 16 * 1024 + ): PtySourceSendReservation | null { + assertPositiveSafeInteger(maxSourceSu, 'maxSourceSu') + const record = this.requireDelivery(identity) + if (record.state !== 'active' && record.state !== 'sealed-unsettled') { + return null + } + if (record.pendingSend) { + throw new Error('PTY source delivery already has a pending send reservation') + } + const remainingWindowSu = record.windowSu - (record.sentEndSu - record.creditedEndSu) + if (remainingWindowSu <= 0 || record.sentEndSu >= record.receivedEndSu) { + return null + } + const containing = record.spans.find( + (span) => span.sourceStartSu <= record.sentEndSu && span.sourceEndSu > record.sentEndSu + ) + if (!containing) { + throw new Error('PTY source delivery cursor is not covered by the retained ledger') + } + const reservedLengthSu = reservedPtySourceSendLength(record, remainingWindowSu, maxSourceSu) + if (reservedLengthSu === null) { + return null + } + const available = reservedLengthSu ?? Math.min(remainingWindowSu, maxSourceSu) + if ( + (!ptySourceSpanIsSplittable(containing) || containing.transform.transformed) && + containing.sourceEndSu - record.sentEndSu > available + ) { + return null + } + const span = sliceForSend(containing, record.sentEndSu, available) + const reservation = Object.freeze({ + reservationId: `pty-source-send:${this.nextReservationId++}`, + identity: record.identity, + span + }) + record.pendingSend = reservation + record.attemptedEndSu = span.sourceEndSu + return reservation + } + + commitSend(reservation: PtySourceSendReservation): void { + const record = this.requireDelivery(reservation.identity) + if (commitPtySourceSend(record, reservation)) { + this.maybeClose(record) + } + } + + rollbackSend(reservation: PtySourceSendReservation): void { + const record = this.requireDelivery(reservation.identity) + rollbackPtySourceSend(record, reservation) + } + + acknowledge(identity: PtySourceDeliveryIdentity, ack: PtySourceCreditAck): PtySourceAckResult { + const record = this.requireDelivery(identity) + const result = acknowledgeDeliveryRecord(record, ack) + if (result === 'advanced') { + this.maybeClose(record) + } + return result + } + + seal(identity: PtySourceDeliveryIdentity): void { + const record = this.requireActive(identity) + record.state = 'sealed-unsettled' + } + + settleExitPublication( + identity: PtySourceDeliveryIdentity, + result: { ok: true } | { ok: false; error: Error } + ): void { + const record = this.requireDelivery(identity) + if (record.state !== 'sealed-unsettled') { + throw new Error('PTY source delivery is not sealed') + } + if (!result.ok) { + return + } + if (record.pendingSend || record.sentEndSu !== record.receivedEndSu) { + throw new Error('PTY source exit cannot publish ahead of preceding source data') + } + record.exitPublished = true + this.maybeClose(record) + } + + cancel( + identity: PtySourceDeliveryIdentity, + reason: string, + replacementDeliveryToken?: string + ): PtySourceDeliveryCancellation { + const record = this.requireDelivery(identity) + record.state = 'closing' + const proof = createDeliveryCancellation(record, reason, replacementDeliveryToken) + this.closeRecord(record) + return proof + } + + closeGeneration(providerGeneration: number): number { + assertPositiveSafeInteger(providerGeneration, 'providerGeneration') + return closeDeliveryGeneration(this.deliveries.values(), providerGeneration, (record) => + this.closeRecord(record) + ) + } + + rotate( + oldIdentity: PtySourceDeliveryIdentity, + newIdentity: PtySourceDeliveryIdentity, + acceptedSourceEndSu: number, + windowSu: number + ): Readonly<{ cancellation: PtySourceDeliveryCancellation; recovery: readonly PtySourceSpan[] }> { + const old = this.requireDelivery(oldIdentity) + const replacementKey = ptySourceDeliveryKey(newIdentity) + if (this.deliveries.has(replacementKey) || this.closedSnapshots.has(replacementKey)) { + throw new Error('PTY source replacement token was already used') + } + const replacement = createReplacementDeliveryRecord( + old, + newIdentity, + acceptedSourceEndSu, + windowSu + ) + this.upstreamOwnerByPty.delete(ptyOwnerKey(old.identity)) + this.deliveries.set(replacementKey, replacement) + this.upstreamOwnerByPty.set(ptyOwnerKey(replacement.identity), replacementKey) + const cancellation = this.cancel(oldIdentity, 'superseded', newIdentity.deliveryToken) + return Object.freeze({ cancellation, recovery: Object.freeze(replacement.spans.slice()) }) + } + + snapshot(identity: PtySourceDeliveryIdentity): PtySourceDeliverySnapshot { + const active = this.deliveries.get(ptySourceDeliveryKey(identity)) + if (active && samePtySourceDelivery(active.identity, identity)) { + return snapshotDeliveryRecord(active) + } + const closed = this.closedSnapshots.get(ptySourceDeliveryKey(identity)) + if (closed && samePtySourceDelivery(closed, identity)) { + return closed + } + throw new Error('Unknown or stale PTY source delivery') + } + + retainedSourceSu = (): number => retainedSourceTotal(this.deliveries.values()) + + retainedDataBytes = (): number => retainedDataBytesTotal(this.deliveries.values()) + + retainedSpans = (): number => retainedSpanTotal(this.deliveries.values()) + + retentionSnapshot(): Readonly<{ + sourceSu: number + dataBytes: number + spans: number + }> { + return Object.freeze({ + sourceSu: this.retainedSourceSu(), + dataBytes: this.retainedDataBytes(), + spans: this.retainedSpans() + }) + } + + private requireActive(identity: PtySourceDeliveryIdentity): DeliveryRecord { + const record = this.requireDelivery(identity) + if (record.state !== 'active') { + throw new Error('PTY source delivery does not admit new source') + } + return record + } + + private requireDelivery(identity: PtySourceDeliveryIdentity): DeliveryRecord { + const record = this.deliveries.get(ptySourceDeliveryKey(identity)) + if (!record || !samePtySourceDelivery(record.identity, identity)) { + throw new Error('Unknown or stale PTY source delivery') + } + return record + } + + private maybeClose(record: DeliveryRecord): void { + if ( + record.state === 'sealed-unsettled' && + record.exitPublished && + record.creditedEndSu === record.receivedEndSu + ) { + this.closeRecord(record) + } + } + + private closeRecord(record: DeliveryRecord): void { + record.state = 'closed' + record.pendingSend = null + record.spans = [] + record.retainedDataBytes = 0 + const key = ptySourceDeliveryKey(record.identity) + const ownerKey = ptyOwnerKey(record.identity) + if (this.upstreamOwnerByPty.get(ownerKey) === key) { + this.upstreamOwnerByPty.delete(ownerKey) + } + this.deliveries.delete(key) + this.closedSnapshots.set(key, snapshotDeliveryRecord(record)) + while (this.closedSnapshots.size > CLOSED_DELIVERY_TOMBSTONE_LIMIT) { + this.closedSnapshots.delete(this.closedSnapshots.keys().next().value!) + } + } +} diff --git a/src/relay/pty-source-credit-limits.ts b/src/relay/pty-source-credit-limits.ts new file mode 100644 index 00000000000..4a271c3d2fa --- /dev/null +++ b/src/relay/pty-source-credit-limits.ts @@ -0,0 +1,39 @@ +import { assertPositiveSafeInteger } from '../shared/pty-source-credit-validation' +import { + DEFAULT_AGGREGATE_RETAINED_DATA_BYTES, + DEFAULT_AGGREGATE_RETAINED_SPANS, + DEFAULT_AGGREGATE_RETAINED_SOURCE_SU, + DEFAULT_RETAINED_DATA_BYTES, + DEFAULT_RETAINED_SPANS, + DEFAULT_RETAINED_SOURCE_SU +} from './pty-source-credit-record' + +export type PtySourceCreditLedgerOptions = { + maxRetainedSourceSu?: number + maxAggregateRetainedSourceSu?: number + maxRetainedDataBytes?: number + maxAggregateRetainedDataBytes?: number + maxRetainedSpans?: number + maxAggregateRetainedSpans?: number +} + +export type PtySourceCreditLimits = Required + +export function resolvePtySourceCreditLimits( + options: PtySourceCreditLedgerOptions +): PtySourceCreditLimits { + const limits = { + maxRetainedSourceSu: options.maxRetainedSourceSu ?? DEFAULT_RETAINED_SOURCE_SU, + maxAggregateRetainedSourceSu: + options.maxAggregateRetainedSourceSu ?? DEFAULT_AGGREGATE_RETAINED_SOURCE_SU, + maxRetainedDataBytes: options.maxRetainedDataBytes ?? DEFAULT_RETAINED_DATA_BYTES, + maxAggregateRetainedDataBytes: + options.maxAggregateRetainedDataBytes ?? DEFAULT_AGGREGATE_RETAINED_DATA_BYTES, + maxRetainedSpans: options.maxRetainedSpans ?? DEFAULT_RETAINED_SPANS, + maxAggregateRetainedSpans: options.maxAggregateRetainedSpans ?? DEFAULT_AGGREGATE_RETAINED_SPANS + } + for (const [name, value] of Object.entries(limits)) { + assertPositiveSafeInteger(value, name) + } + return Object.freeze(limits) +} diff --git a/src/relay/pty-source-credit-record.ts b/src/relay/pty-source-credit-record.ts new file mode 100644 index 00000000000..cbdfd497e2d --- /dev/null +++ b/src/relay/pty-source-credit-record.ts @@ -0,0 +1,268 @@ +import type { + PtySourceDeliveryIdentity, + PtySourceDeliveryCancellation, + PtySourceDeliverySnapshot, + PtySourceSpan, + PtySourceTransform +} from '../shared/pty-source-credit-contract' +import { ptySourceSpanIsSplittable } from '../shared/pty-source-credit-contract' +import { + assertNonNegativeSafeInteger, + assertPositiveSafeInteger, + assertPtySourceIdentity, + assertPtySourceSpan +} from '../shared/pty-source-credit-validation' +import { chargedPtyRetainedStringBytes } from '../shared/pty-retained-string-memory' + +export const DEFAULT_RETAINED_SOURCE_SU = 512 * 1024 +export const DEFAULT_AGGREGATE_RETAINED_SOURCE_SU = 48 * 1024 * 1024 +export const DEFAULT_RETAINED_DATA_BYTES = 2 * 1024 * 1024 +export const DEFAULT_AGGREGATE_RETAINED_DATA_BYTES = 64 * 1024 * 1024 +export const DEFAULT_RETAINED_SPANS = 1_024 +export const DEFAULT_AGGREGATE_RETAINED_SPANS = 64 * 1_024 +export const MAX_SOURCE_SPAN_DATA_BYTES = 1024 * 1024 +export const CLOSED_DELIVERY_TOMBSTONE_LIMIT = 256 + +export type PtySourceSendReservation = Readonly<{ + reservationId: string + identity: PtySourceDeliveryIdentity + span: PtySourceSpan +}> + +export type DeliveryRecord = { + identity: PtySourceDeliveryIdentity + state: 'active' | 'sealed-unsettled' | 'closing' | 'closed' + windowSu: number + receivedEndSu: number + sentEndSu: number + creditedEndSu: number + retainedDataBytes: number + spans: PtySourceSpan[] + sentBoundaries: Set + pendingSend: PtySourceSendReservation | null + reservedAckEndSu: number | null + attemptedEndSu: number | null + exitPublished: boolean + generationClosed: boolean +} + +export function ptyOwnerKey(identity: PtySourceDeliveryIdentity): string { + return `${identity.providerGeneration}\0${identity.id}\0${identity.ptyIncarnation}` +} + +export function createDeliveryRecord( + identity: PtySourceDeliveryIdentity, + windowSu: number, + checkpointSourceEndSu: number +): DeliveryRecord { + return { + identity: Object.freeze({ ...identity }), + state: 'active', + windowSu, + receivedEndSu: checkpointSourceEndSu, + sentEndSu: checkpointSourceEndSu, + creditedEndSu: checkpointSourceEndSu, + retainedDataBytes: 0, + spans: [], + sentBoundaries: new Set([checkpointSourceEndSu]), + pendingSend: null, + reservedAckEndSu: null, + attemptedEndSu: null, + exitPublished: false, + generationClosed: false + } +} + +export type PtySourceAppendInput = Readonly<{ + spanId: string + data: string + displayStart: number + displayEnd: number + splittable: boolean + transform: PtySourceTransform +}> + +export function createAppendedSourceSpan( + record: DeliveryRecord, + input: PtySourceAppendInput +): PtySourceSpan { + const span = Object.freeze({ + ...record.identity, + ...input, + transform: Object.freeze({ ...input.transform }), + sourceStartSu: record.receivedEndSu, + sourceEndSu: record.receivedEndSu + input.transform.rawLengthSu + }) + assertPtySourceSpan(span) + return span +} + +export function sliceAtSourceStart(span: PtySourceSpan, sourceStartSu: number): PtySourceSpan { + if (sourceStartSu === span.sourceStartSu) { + return span + } + if (!ptySourceSpanIsSplittable(span) || span.transform.transformed) { + throw new Error('Indivisible PTY source span cannot be split for recovery') + } + const offset = sourceStartSu - span.sourceStartSu + return Object.freeze({ + ...span, + spanId: `${span.spanId}:suffix:${sourceStartSu}`, + sourceStartSu, + displayStart: span.displayStart + offset, + data: span.data.slice(offset), + transform: Object.freeze({ + ...span.transform, + rawLengthSu: span.sourceEndSu - sourceStartSu + }) + }) +} + +export function sliceForSend( + span: PtySourceSpan, + sourceStartSu: number, + maxSourceSu: number +): PtySourceSpan { + const remaining = sliceAtSourceStart(span, sourceStartSu) + const sourceLengthSu = remaining.sourceEndSu - remaining.sourceStartSu + if (sourceLengthSu <= maxSourceSu) { + return remaining + } + if (!ptySourceSpanIsSplittable(remaining) || remaining.transform.transformed) { + throw new Error('Indivisible PTY source span does not fit the available source window') + } + let endOffset = maxSourceSu + const trailing = remaining.data.charCodeAt(endOffset - 1) + const following = remaining.data.charCodeAt(endOffset) + if ( + endOffset > 0 && + trailing >= 0xd800 && + trailing <= 0xdbff && + following >= 0xdc00 && + following <= 0xdfff + ) { + endOffset-- + } + if (endOffset <= 0) { + throw new Error('Available source window would split a surrogate pair') + } + return Object.freeze({ + ...remaining, + spanId: `${remaining.spanId}:slice:${remaining.sourceStartSu + endOffset}`, + sourceEndSu: remaining.sourceStartSu + endOffset, + displayEnd: remaining.displayStart + endOffset, + data: remaining.data.slice(0, endOffset), + transform: Object.freeze({ ...remaining.transform, rawLengthSu: endOffset }) + }) +} + +export function snapshotDeliveryRecord(record: DeliveryRecord): PtySourceDeliverySnapshot { + return Object.freeze({ + ...record.identity, + state: record.state, + windowSu: record.windowSu, + receivedEndSu: record.receivedEndSu, + sentEndSu: record.sentEndSu, + creditedEndSu: record.creditedEndSu, + exitPublished: record.exitPublished, + generationClosed: record.generationClosed + }) +} + +export function retainedSourceTotal(records: Iterable): number { + let total = 0 + for (const record of records) { + total += record.receivedEndSu - record.creditedEndSu + } + return total +} + +export function retainedDataBytesTotal(records: Iterable): number { + let total = 0 + for (const record of records) { + total += record.retainedDataBytes + } + return total +} + +export function retainedSpanTotal(records: Iterable): number { + let total = 0 + for (const record of records) { + total += record.spans.length + } + return total +} + +export function createReplacementDeliveryRecord( + old: DeliveryRecord, + newIdentity: PtySourceDeliveryIdentity, + acceptedSourceEndSu: number, + windowSu: number +): DeliveryRecord { + assertPtySourceIdentity(newIdentity) + assertPositiveSafeInteger(windowSu, 'windowSu') + assertNonNegativeSafeInteger(acceptedSourceEndSu, 'acceptedSourceEndSu') + const committedCheckpoint = + acceptedSourceEndSu <= old.sentEndSu && old.sentBoundaries.has(acceptedSourceEndSu) + const attemptedCheckpoint = acceptedSourceEndSu === old.attemptedEndSu + if ( + acceptedSourceEndSu < old.creditedEndSu || + (!committedCheckpoint && !attemptedCheckpoint) || + old.pendingSend || + newIdentity.id !== old.identity.id || + newIdentity.ptyIncarnation !== old.identity.ptyIncarnation || + newIdentity.providerGeneration !== old.identity.providerGeneration + ) { + throw new Error('PTY source recovery checkpoint does not exactly cover the retained delivery') + } + if (acceptedSourceEndSu > old.sentEndSu) { + old.sentEndSu = acceptedSourceEndSu + old.sentBoundaries.add(acceptedSourceEndSu) + } + old.attemptedEndSu = null + const replacement = createDeliveryRecord(newIdentity, windowSu, acceptedSourceEndSu) + replacement.state = old.state === 'sealed-unsettled' ? 'sealed-unsettled' : 'active' + replacement.receivedEndSu = old.receivedEndSu + replacement.spans = old.spans + .filter((span) => span.sourceEndSu > acceptedSourceEndSu) + .map((span) => sliceAtSourceStart(span, Math.max(span.sourceStartSu, acceptedSourceEndSu))) + .map((span) => Object.freeze({ ...span, ...replacement.identity })) + replacement.retainedDataBytes = replacement.spans.reduce( + (bytes, span) => bytes + chargedPtyRetainedStringBytes(span.data), + 0 + ) + return replacement +} + +export function createDeliveryCancellation( + record: DeliveryRecord, + reason: string, + replacementDeliveryToken?: string +): PtySourceDeliveryCancellation { + return Object.freeze({ + ...record.identity, + reason, + sentEndSu: record.sentEndSu, + creditedEndSu: record.creditedEndSu, + remainingStartSu: record.creditedEndSu, + remainingEndSu: record.sentEndSu, + ...(replacementDeliveryToken ? { replacementDeliveryToken } : {}) + }) +} + +export function closeDeliveryGeneration( + records: Iterable, + providerGeneration: number, + close: (record: DeliveryRecord) => void +): number { + let closed = 0 + for (const record of Array.from(records)) { + if (record.identity.providerGeneration !== providerGeneration || record.state === 'closed') { + continue + } + record.generationClosed = true + close(record) + closed++ + } + return closed +} diff --git a/src/relay/pty-source-credit-scheduler.test.ts b/src/relay/pty-source-credit-scheduler.test.ts new file mode 100644 index 00000000000..1e1c2ab76e6 --- /dev/null +++ b/src/relay/pty-source-credit-scheduler.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from 'vitest' +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import { RelayPtySourceCreditLedger } from './pty-source-credit-ledger' +import { RelayPtySourceCreditScheduler } from './pty-source-credit-scheduler' + +function identity(index: number): PtySourceDeliveryIdentity { + return { + id: `pty-${index}`, + providerGeneration: 1, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: `incarnation-${index}`, + deliveryToken: `token-${index}` + } +} + +describe('RelayPtySourceCreditScheduler', () => { + it('round-robins fifty continuously active PTYs without exceeding turn budgets', () => { + const ledger = new RelayPtySourceCreditLedger() + const scheduler = new RelayPtySourceCreditScheduler(ledger) + const identities = Array.from({ length: 50 }, (_, index) => identity(index)) + for (const owner of identities) { + ledger.open(owner, 1024) + ledger.append(owner, { + spanId: `span-${owner.id}`, + data: 'x'.repeat(100), + displayStart: 0, + displayEnd: 100, + splittable: true, + transform: { transformed: false, rawLengthSu: 100, scalarSafe: true } + }) + scheduler.enqueue(owner) + } + + const visited: string[] = [] + for (let turn = 0; turn < 25; turn++) { + const reservations = scheduler.takeTurn() + expect(reservations.length).toBeLessThanOrEqual(2) + expect( + reservations.reduce( + (total, reservation) => + total + reservation.span.sourceEndSu - reservation.span.sourceStartSu, + 0 + ) + ).toBeLessThanOrEqual(32 * 1024) + for (const reservation of reservations) { + visited.push(reservation.identity.id) + ledger.commitSend(reservation) + } + } + + expect(new Set(visited)).toEqual(new Set(identities.map((owner) => owner.id))) + }) + + it('does not duplicate a turn after remove and re-enqueue', () => { + const ledger = new RelayPtySourceCreditLedger() + const scheduler = new RelayPtySourceCreditScheduler(ledger) + const owner = identity(1) + ledger.open(owner, 8) + ledger.append(owner, { + spanId: 'span-1', + data: 'ab', + displayStart: 0, + displayEnd: 2, + splittable: true, + transform: { transformed: false, rawLengthSu: 2, scalarSafe: true } + }) + scheduler.enqueue(owner) + scheduler.remove(owner) + scheduler.enqueue(owner) + + expect(scheduler.takeTurn(2, 2)).toHaveLength(1) + }) + + it('isolates one rejecting delivery without losing reservations or blocking peers', () => { + const ledger = new RelayPtySourceCreditLedger() + const scheduler = new RelayPtySourceCreditScheduler(ledger) + const owners = [identity(1), identity(2), identity(3)] + for (const owner of owners) { + ledger.open(owner, 8) + ledger.append(owner, { + spanId: `span-${owner.id}`, + data: 'ab', + displayStart: 0, + displayEnd: 2, + splittable: true, + transform: { transformed: false, rawLengthSu: 2, scalarSafe: true } + }) + scheduler.enqueue(owner) + } + const reserveNextSend = ledger.reserveNextSend.bind(ledger) + const reserve = vi.spyOn(ledger, 'reserveNextSend').mockImplementation((owner, maxSourceSu) => { + if (owner === owners[1]) { + throw new Error('delivery ledger corrupted') + } + return reserveNextSend(owner, maxSourceSu) + }) + + const reservations = scheduler.takeTurn(2, 4) + + expect(reserve).toHaveBeenCalledTimes(3) + expect(reservations.map(({ identity: owner }) => owner.id)).toEqual(['pty-1', 'pty-3']) + for (const reservation of reservations) { + ledger.commitSend(reservation) + } + scheduler.takeTurn(2, 4) + expect(reserve).toHaveBeenCalledTimes(5) + expect(reserve.mock.calls.map(([owner]) => owner.id)).toEqual([ + 'pty-1', + 'pty-2', + 'pty-3', + 'pty-1', + 'pty-3' + ]) + }) +}) diff --git a/src/relay/pty-source-credit-scheduler.ts b/src/relay/pty-source-credit-scheduler.ts new file mode 100644 index 00000000000..792daec0ff4 --- /dev/null +++ b/src/relay/pty-source-credit-scheduler.ts @@ -0,0 +1,70 @@ +import { + ptySourceDeliveryKey, + type PtySourceDeliveryIdentity +} from '../shared/pty-source-credit-contract' +import type { + RelayPtySourceCreditLedger, + PtySourceSendReservation +} from './pty-source-credit-ledger' + +export const PTY_SOURCE_SCHEDULER_MAX_FRAMES = 2 +export const PTY_SOURCE_SCHEDULER_MAX_SU = 32 * 1024 + +export class RelayPtySourceCreditScheduler { + private readonly identities = new Map() + private readonly readyKeys: string[] = [] + private readonly queuedKeys = new Set() + + constructor(private readonly ledger: RelayPtySourceCreditLedger) {} + + enqueue(identity: PtySourceDeliveryIdentity): void { + const key = ptySourceDeliveryKey(identity) + this.identities.set(key, identity) + if (!this.queuedKeys.has(key)) { + this.queuedKeys.add(key) + this.readyKeys.push(key) + } + } + + remove(identity: PtySourceDeliveryIdentity): void { + const key = ptySourceDeliveryKey(identity) + this.identities.delete(key) + this.queuedKeys.delete(key) + for (let index = this.readyKeys.length - 1; index >= 0; index--) { + if (this.readyKeys[index] === key) { + this.readyKeys.splice(index, 1) + } + } + } + + takeTurn( + maxFrames = PTY_SOURCE_SCHEDULER_MAX_FRAMES, + maxSourceSu = PTY_SOURCE_SCHEDULER_MAX_SU + ): PtySourceSendReservation[] { + const reservations: PtySourceSendReservation[] = [] + let remainingKeys = this.readyKeys.length + let admittedSu = 0 + while (remainingKeys-- > 0 && reservations.length < maxFrames && admittedSu < maxSourceSu) { + const key = this.readyKeys.shift()! + this.queuedKeys.delete(key) + const identity = this.identities.get(key) + if (!identity) { + continue + } + let reservation: PtySourceSendReservation | null + try { + reservation = this.ledger.reserveNextSend(identity, maxSourceSu - admittedSu) + } catch { + this.identities.delete(key) + continue + } + this.enqueue(identity) + if (!reservation) { + continue + } + reservations.push(reservation) + admittedSu += reservation.span.sourceEndSu - reservation.span.sourceStartSu + } + return reservations + } +} diff --git a/src/relay/pty-source-credit-settlement.ts b/src/relay/pty-source-credit-settlement.ts new file mode 100644 index 00000000000..464ea9cefe1 --- /dev/null +++ b/src/relay/pty-source-credit-settlement.ts @@ -0,0 +1,112 @@ +import type { PtySourceCreditAck } from '../shared/pty-source-credit-contract' +import { assertPtySourceAck } from '../shared/pty-source-credit-validation' +import { chargedPtyRetainedStringBytes } from '../shared/pty-retained-string-memory' +import type { DeliveryRecord, PtySourceSendReservation } from './pty-source-credit-record' + +export type PtySourceAckResult = 'advanced' | 'reserved' | 'duplicate' | 'regression' + +export function reservedPtySourceSendLength( + record: DeliveryRecord, + remainingWindowSu: number, + maxSourceSu: number +): number | null | undefined { + if (record.reservedAckEndSu === null) { + return undefined + } + const lengthSu = record.reservedAckEndSu - record.sentEndSu + return lengthSu > 0 && lengthSu <= remainingWindowSu && lengthSu <= maxSourceSu ? lengthSu : null +} + +function reclaimCreditedSpans(record: DeliveryRecord): void { + while (record.spans[0]?.sourceEndSu <= record.creditedEndSu) { + record.retainedDataBytes -= chargedPtyRetainedStringBytes(record.spans.shift()!.data) + } +} + +function advanceCredit(record: DeliveryRecord, creditedEndSu: number): void { + record.creditedEndSu = creditedEndSu + for (const boundary of record.sentBoundaries) { + if (boundary < creditedEndSu) { + record.sentBoundaries.delete(boundary) + } + } + reclaimCreditedSpans(record) +} + +export function acknowledgeDeliveryRecord( + record: DeliveryRecord, + ack: PtySourceCreditAck +): PtySourceAckResult { + assertPtySourceAck(ack) + if (record.state === 'closed' || record.state === 'closing') { + throw new Error('PTY source ACK targets a closed delivery') + } + if ( + ack.id !== record.identity.id || + ack.clientGeneration !== record.identity.clientGeneration || + ack.ownerGeneration !== record.identity.ownerGeneration || + ack.deliveryToken !== record.identity.deliveryToken + ) { + throw new Error('PTY source ACK does not own this delivery') + } + const reservedBoundarySu = record.pendingSend?.span.sourceEndSu ?? record.reservedAckEndSu + const eligibleEndSu = reservedBoundarySu ?? record.sentEndSu + if (ack.creditedEndSu > eligibleEndSu) { + throw new Error('PTY source ACK exceeds sent source credit') + } + if (ack.creditedEndSu === record.creditedEndSu) { + return 'duplicate' + } + if (ack.creditedEndSu < record.creditedEndSu) { + return 'regression' + } + if (ack.creditedEndSu > record.sentEndSu) { + if (ack.creditedEndSu !== reservedBoundarySu) { + throw new Error('PTY source ACK does not match a reserved send boundary') + } + record.reservedAckEndSu = ack.creditedEndSu + return 'reserved' + } + if (!record.sentBoundaries.has(ack.creditedEndSu)) { + throw new Error('PTY source ACK does not match a committed send boundary') + } + advanceCredit(record, ack.creditedEndSu) + return 'advanced' +} + +export function settleReservedPtySourceAck(record: DeliveryRecord): boolean { + const creditedEndSu = record.reservedAckEndSu + record.reservedAckEndSu = null + if (creditedEndSu === null) { + return false + } + if (creditedEndSu !== record.sentEndSu || !record.sentBoundaries.has(creditedEndSu)) { + throw new Error('PTY source reserved ACK does not match the settled send boundary') + } + advanceCredit(record, creditedEndSu) + return true +} + +export function commitPtySourceSend( + record: DeliveryRecord, + reservation: PtySourceSendReservation +): boolean { + if (record.pendingSend !== reservation) { + throw new Error('PTY source send reservation is stale') + } + record.pendingSend = null + record.sentEndSu = reservation.span.sourceEndSu + record.sentBoundaries.add(record.sentEndSu) + record.attemptedEndSu = null + return settleReservedPtySourceAck(record) +} + +export function rollbackPtySourceSend( + record: DeliveryRecord, + reservation: PtySourceSendReservation +): void { + if (record.pendingSend !== reservation) { + return + } + record.pendingSend = null +} diff --git a/src/relay/pty-source-delivery-ownership.ts b/src/relay/pty-source-delivery-ownership.ts new file mode 100644 index 00000000000..b7f1306e72a --- /dev/null +++ b/src/relay/pty-source-delivery-ownership.ts @@ -0,0 +1,18 @@ +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import type { PtyConsumerSessionGrant } from '../shared/pty-consumer-session' + +export function ownedPtySourceDelivery( + identity: PtySourceDeliveryIdentity | undefined, + grant: Readonly, + id: string +): PtySourceDeliveryIdentity | null { + if ( + !identity || + identity.id !== id || + identity.clientGeneration !== grant.clientGeneration || + identity.ownerGeneration !== grant.ownerGeneration + ) { + return null + } + return identity +} diff --git a/src/relay/pty-source-replay-index.test.ts b/src/relay/pty-source-replay-index.test.ts new file mode 100644 index 00000000000..efbb17451b0 --- /dev/null +++ b/src/relay/pty-source-replay-index.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest' +import type { PtySourceSpan } from '../shared/pty-source-credit-contract' +import { PtySourceReplayIndex } from './pty-source-replay-index' + +function span(id: string, data: string, sourceStartSu: number): PtySourceSpan { + return { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + spanId: id, + sourceStartSu, + sourceEndSu: sourceStartSu + data.length, + displayStart: sourceStartSu, + displayEnd: sourceStartSu + data.length, + data, + splittable: true, + transform: { transformed: false, rawLengthSu: data.length, scalarSafe: true } + } +} + +describe('PtySourceReplayIndex', () => { + it('wraps the existing chunked buffer and preserves exact retained source ranges', () => { + const replay = new PtySourceReplayIndex(6) + replay.append(span('one', 'abcd', 0)) + replay.append(span('two', 'efgh', 4)) + + expect(replay.readLegacy()).toBe('cdefgh') + expect(replay.recoveryFrom(3)).toEqual({ + ok: true, + records: [ + expect.objectContaining({ sourceStartSu: 3, data: 'd' }), + expect.objectContaining({ sourceStartSu: 4, data: 'efgh' }) + ], + liveEndSourceSu: 8 + }) + expect(replay.recoveryFrom(1)).toMatchObject({ + ok: false, + reason: 'checkpoint-before-retained-range' + }) + }) + + it('does not wall-clock dedupe identical replay payloads', () => { + const first = new PtySourceReplayIndex(16) + const second = new PtySourceReplayIndex(16) + first.append(span('first', 'same', 0)) + second.append({ ...span('second', 'same', 0), deliveryToken: 'token-2' }) + + expect(first.recoveryFrom(0)).toMatchObject({ ok: true }) + expect(second.recoveryFrom(0)).toMatchObject({ ok: true }) + }) + + it('preserves indivisible-form split metadata during recovery', () => { + const replay = new PtySourceReplayIndex(16) + replay.append({ + ...span('range', 'abcd', 0), + splittable: undefined, + indivisible: false + }) + + expect(replay.recoveryFrom(2)).toMatchObject({ + ok: true, + records: [expect.objectContaining({ sourceStartSu: 2, data: 'cd' })] + }) + }) + + it('rejects replay checkpoints and retained heads inside a scalar', () => { + const replay = new PtySourceReplayIndex(4) + replay.append(span('range', 'a😀b', 0)) + expect(replay.recoveryFrom(2)).toMatchObject({ + ok: false, + reason: 'checkpoint-inside-scalar' + }) + + const trimmed = new PtySourceReplayIndex(1) + trimmed.append(span('emoji', '😀', 0)) + expect(trimmed.recoveryFrom(1)).toMatchObject({ + ok: false, + reason: 'checkpoint-before-retained-range', + retainedStartSourceSu: 2 + }) + }) + + it('rejects a recovery checkpoint beyond the live source range', () => { + const replay = new PtySourceReplayIndex(16) + replay.append(span('range', 'abcd', 0)) + + expect(replay.recoveryFrom(5)).toMatchObject({ + ok: false, + reason: 'checkpoint-after-live-range', + liveEndSourceSu: 4 + }) + }) +}) diff --git a/src/relay/pty-source-replay-index.ts b/src/relay/pty-source-replay-index.ts new file mode 100644 index 00000000000..1dfe5596eb1 --- /dev/null +++ b/src/relay/pty-source-replay-index.ts @@ -0,0 +1,201 @@ +import { RecentPtyOutputBuffer } from '../main/runtime/recent-pty-output-buffer' +import { ptySourceSpanIsSplittable, type PtySourceSpan } from '../shared/pty-source-credit-contract' +import { + assertNonNegativeSafeInteger, + assertPtySourceSpan +} from '../shared/pty-source-credit-validation' + +export type PtySourceReplayRecord = Readonly< + Pick< + PtySourceSpan, + | 'spanId' + | 'sourceStartSu' + | 'sourceEndSu' + | 'displayStart' + | 'displayEnd' + | 'data' + | 'splittable' + | 'indivisible' + | 'transform' + > +> + +export type PtySourceReplayResult = + | Readonly<{ ok: true; records: readonly PtySourceReplayRecord[]; liveEndSourceSu: number }> + | Readonly<{ + ok: false + reason: + | 'checkpoint-before-retained-range' + | 'checkpoint-after-live-range' + | 'checkpoint-inside-indivisible-transform' + | 'checkpoint-inside-scalar' + retainedStartSourceSu: number + liveEndSourceSu: number + }> + +export class PtySourceReplayIndex { + private readonly output: RecentPtyOutputBuffer + private readonly records: PtySourceReplayRecord[] = [] + private displayEnd = 0 + private retainedStartSourceSu = 0 + private liveEndSourceSu = 0 + + constructor(limit: number) { + this.output = new RecentPtyOutputBuffer({ preserveChunkBoundaries: false, limit }) + } + + append(span: PtySourceSpan): void { + assertPtySourceSpan(span) + if ( + span.sourceStartSu !== this.liveEndSourceSu || + span.displayStart !== this.displayEnd || + span.displayEnd - span.displayStart !== span.data.length + ) { + throw new Error('PTY replay source/display ranges must be contiguous') + } + this.output.append(span.data) + this.records.push( + Object.freeze({ + spanId: span.spanId, + sourceStartSu: span.sourceStartSu, + sourceEndSu: span.sourceEndSu, + displayStart: span.displayStart, + displayEnd: span.displayEnd, + data: span.data, + splittable: span.splittable, + indivisible: span.indivisible, + transform: span.transform + }) + ) + this.displayEnd = span.displayEnd + this.liveEndSourceSu = span.sourceEndSu + this.evictUnmappedPrefix() + } + + readLegacy(): string { + return this.output.read() + } + + recoveryFrom(checkpointSourceEndSu: number): PtySourceReplayResult { + assertNonNegativeSafeInteger(checkpointSourceEndSu, 'checkpointSourceEndSu') + if (checkpointSourceEndSu < this.retainedStartSourceSu) { + return Object.freeze({ + ok: false, + reason: 'checkpoint-before-retained-range', + retainedStartSourceSu: this.retainedStartSourceSu, + liveEndSourceSu: this.liveEndSourceSu + }) + } + if (checkpointSourceEndSu > this.liveEndSourceSu) { + return Object.freeze({ + ok: false, + reason: 'checkpoint-after-live-range', + retainedStartSourceSu: this.retainedStartSourceSu, + liveEndSourceSu: this.liveEndSourceSu + }) + } + const containing = this.records.find( + (record) => + record.sourceStartSu < checkpointSourceEndSu && checkpointSourceEndSu < record.sourceEndSu + ) + if ( + containing?.transform.transformed || + (containing && !ptySourceSpanIsSplittable(containing)) + ) { + return Object.freeze({ + ok: false, + reason: 'checkpoint-inside-indivisible-transform', + retainedStartSourceSu: this.retainedStartSourceSu, + liveEndSourceSu: this.liveEndSourceSu + }) + } + if ( + containing && + splitsSurrogatePair(containing.data, checkpointSourceEndSu - containing.sourceStartSu) + ) { + return Object.freeze({ + ok: false, + reason: 'checkpoint-inside-scalar', + retainedStartSourceSu: this.retainedStartSourceSu, + liveEndSourceSu: this.liveEndSourceSu + }) + } + const records = this.records + .filter((record) => record.sourceEndSu > checkpointSourceEndSu) + .map((record) => this.sliceRecord(record, checkpointSourceEndSu)) + return Object.freeze({ + ok: true, + records: Object.freeze(records), + liveEndSourceSu: this.liveEndSourceSu + }) + } + + private evictUnmappedPrefix(): void { + const retainedDisplayStart = this.displayEnd - this.output.read().length + while (this.records[0]?.displayEnd <= retainedDisplayStart) { + const removed = this.records.shift()! + this.retainedStartSourceSu = removed.sourceEndSu + } + const head = this.records[0] + if (!head || head.displayStart >= retainedDisplayStart) { + return + } + if (head.transform.transformed || !ptySourceSpanIsSplittable(head)) { + this.records.shift() + this.retainedStartSourceSu = head.sourceEndSu + return + } + const displayOffset = retainedDisplayStart - head.displayStart + if (splitsSurrogatePair(head.data, displayOffset)) { + this.records.shift() + this.retainedStartSourceSu = head.sourceEndSu + return + } + const sourceStartSu = head.sourceStartSu + displayOffset + this.records[0] = Object.freeze({ + ...head, + spanId: `${head.spanId}:retained:${sourceStartSu}`, + sourceStartSu, + displayStart: retainedDisplayStart, + data: head.data.slice(displayOffset), + transform: Object.freeze({ + ...head.transform, + rawLengthSu: head.sourceEndSu - sourceStartSu + }) + }) + this.retainedStartSourceSu = sourceStartSu + } + + private sliceRecord( + record: PtySourceReplayRecord, + checkpointSourceEndSu: number + ): PtySourceReplayRecord { + if (checkpointSourceEndSu <= record.sourceStartSu) { + return record + } + const offset = checkpointSourceEndSu - record.sourceStartSu + return Object.freeze({ + ...record, + spanId: `${record.spanId}:recovery:${checkpointSourceEndSu}`, + sourceStartSu: checkpointSourceEndSu, + displayStart: record.displayStart + offset, + data: record.data.slice(offset), + transform: Object.freeze({ + ...record.transform, + rawLengthSu: record.sourceEndSu - checkpointSourceEndSu + }) + }) + } +} + +function splitsSurrogatePair(data: string, offset: number): boolean { + const preceding = data.charCodeAt(offset - 1) + const following = data.charCodeAt(offset) + return ( + offset > 0 && + preceding >= 0xd800 && + preceding <= 0xdbff && + following >= 0xdc00 && + following <= 0xdfff + ) +} diff --git a/src/relay/relay-frame-decoder.ts b/src/relay/relay-frame-decoder.ts new file mode 100644 index 00000000000..a2c898f017e --- /dev/null +++ b/src/relay/relay-frame-decoder.ts @@ -0,0 +1,264 @@ +import { + containFrameDecoderContinuation, + publishFrameDecoderError, + type DecodedFrame, + type FrameDecoderOptions +} from '../shared/relay-frame-decoder-contract' +import { RelayFrameBuffer } from '../shared/relay-frame-buffer' +export { + FrameDecoderContinuationError, + type DecodedFrame, + type FrameDecoderOptions +} from '../shared/relay-frame-decoder-contract' + +export const HEADER_LENGTH = 13 +export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 +export const FRAME_DECODER_MAX_FRAMES_PER_TURN = 64 +export const FRAME_DECODER_MAX_BYTES_PER_TURN = MAX_MESSAGE_SIZE + HEADER_LENGTH +export const FRAME_DECODER_MAX_TURN_MS = 4, + FRAME_DECODER_MAX_RETAINED_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024 + +export class FrameDecoder { + private readonly buffer = new RelayFrameBuffer() + private oversizedPayloadBytesRemaining = 0 + private onFrame: (frame: DecodedFrame) => void + private onError: (err: Error) => void + private maxFramesPerTurn: number + private maxBytesPerTurn: number + private maxTurnMs: number + private now: () => number + private schedule: (callback: () => void) => unknown + private cancelScheduled: (handle: unknown) => void + private pause: (() => void) | null + private resume: (() => void) | null + private continuationHandle: unknown + private continuationHandleAssigned = false + private continuationScheduled = false + private paused = false + private draining = false + private generation = 0 + + constructor( + onFrame: (frame: DecodedFrame) => void, + onError?: (err: Error) => void, + options: FrameDecoderOptions = {} + ) { + this.onFrame = onFrame + this.onError = onError ?? ((error) => process.stderr.write(`[relay] ${error.message}\n`)) + this.maxFramesPerTurn = positiveLimit( + options.maxFramesPerTurn, + FRAME_DECODER_MAX_FRAMES_PER_TURN + ) + this.maxBytesPerTurn = positiveLimit(options.maxBytesPerTurn, FRAME_DECODER_MAX_BYTES_PER_TURN) + this.maxTurnMs = positiveLimit(options.maxTurnMs, FRAME_DECODER_MAX_TURN_MS) + this.now = options.now ?? Date.now + this.schedule = options.schedule ?? ((callback) => setImmediate(callback)) + this.cancelScheduled = + options.cancelScheduled ?? ((handle) => clearImmediate(handle as NodeJS.Immediate)) + this.pause = options.pause ?? null + this.resume = options.resume ?? null + } + + feed(chunk: Buffer | Uint8Array): void { + const buf = Buffer.isBuffer(chunk) + ? chunk + : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength) + const retained = this.buffer.length + buf.length + if (retained > FRAME_DECODER_MAX_RETAINED_BYTES) { + this.reset() + publishFrameDecoderError( + this.onError, + new Error(`Frame decoder retained-input limit exceeded: ${retained}`) + ) + return + } + if (buf.length > 0) { + this.buffer.append(buf) + } + if (!this.draining && !this.continuationScheduled) { + this.drainTurn() + } + } + + reset(): void { + this.generation += 1 + this.cancelContinuation() + this.buffer.clear() + this.oversizedPayloadBytesRemaining = 0 + this.releasePause() + } + + drain(): Buffer { + const out = this.buffer.drain() + this.reset() + return out + } + + private drainTurn(): void { + if (this.draining) { + return + } + this.draining = true + const generation = this.generation + const startedAt = this.now() + let frames = 0 + let bytes = 0 + + try { + while (generation === this.generation) { + if ( + frames >= this.maxFramesPerTurn || + bytes >= this.maxBytesPerTurn || + (frames > 0 && this.now() - startedAt >= this.maxTurnMs) + ) { + break + } + const discarded = this.discardOversizedPayload(bytes) + if (discarded > 0) { + bytes += discarded + continue + } + if (this.buffer.length < HEADER_LENGTH) { + break + } + const header = this.buffer.peek(HEADER_LENGTH) + const length = header.readUInt32BE(9) + if (length > MAX_MESSAGE_SIZE) { + this.buffer.discard(HEADER_LENGTH) + this.oversizedPayloadBytesRemaining = length + bytes += HEADER_LENGTH + publishFrameDecoderError( + this.onError, + new Error(`Frame payload too large: ${length} bytes — discarded`) + ) + continue + } + const totalLength = HEADER_LENGTH + length + if (this.buffer.length < totalLength) { + break + } + if (frames > 0 && bytes + totalLength > this.maxBytesPerTurn) { + break + } + const framed = this.buffer.take(totalLength) + frames += 1 + bytes += totalLength + this.onFrame({ + type: framed[0], + id: framed.readUInt32BE(1), + ack: framed.readUInt32BE(5), + payload: framed.subarray(HEADER_LENGTH, totalLength) + }) + } + } finally { + this.draining = false + } + + if (generation !== this.generation) { + return + } + if (this.hasRunnableWork()) { + this.scheduleContinuation() + } else { + this.releasePause() + } + } + + private discardOversizedPayload(bytes: number): number { + if (this.oversizedPayloadBytesRemaining === 0 || this.buffer.length === 0) { + return 0 + } + const discarded = Math.min( + this.oversizedPayloadBytesRemaining, + this.buffer.length, + Math.max(1, this.maxBytesPerTurn - bytes) + ) + this.buffer.discard(discarded) + this.oversizedPayloadBytesRemaining -= discarded + return discarded + } + + private hasRunnableWork(): boolean { + if (this.oversizedPayloadBytesRemaining > 0) { + return this.buffer.length > 0 + } + if (this.buffer.length < HEADER_LENGTH) { + return false + } + const length = this.buffer.peek(HEADER_LENGTH).readUInt32BE(9) + return length > MAX_MESSAGE_SIZE || this.buffer.length >= HEADER_LENGTH + length + } + + private scheduleContinuation(): void { + if (this.continuationScheduled) { + return + } + const generation = this.generation + this.continuationScheduled = true + try { + this.acquirePause() + } catch (error) { + this.continuationScheduled = false + throw error + } + if (generation !== this.generation) { + this.continuationScheduled = false + return + } + try { + this.continuationHandle = this.schedule(() => { + if (!this.continuationScheduled || generation !== this.generation) { + return + } + this.continuationScheduled = false + this.continuationHandleAssigned = false + this.continuationHandle = undefined + try { + this.drainTurn() + } catch (error) { + containFrameDecoderContinuation(() => this.reset(), this.onError, error) + } + }) + this.continuationHandleAssigned = true + } catch (error) { + this.continuationScheduled = false + this.continuationHandle = undefined + this.releasePause() + throw error + } + } + + private cancelContinuation(): void { + if (!this.continuationScheduled) { + return + } + this.continuationScheduled = false + if (this.continuationHandleAssigned) { + this.cancelScheduled(this.continuationHandle) + } + this.continuationHandleAssigned = false + this.continuationHandle = undefined + } + + private acquirePause(): void { + if (!this.paused) { + this.paused = true + try { + this.pause?.() + } catch (error) { + this.paused = false + throw error + } + } + } + + private releasePause(): void { + if (this.paused) { + this.paused = false + this.resume?.() + } + } +} + +const positiveLimit = (value: number | undefined, fallback: number): number => + value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback diff --git a/src/relay/relay-handshake-roundtrip.test.ts b/src/relay/relay-handshake-roundtrip.test.ts index bc7ba71c19d..0713d62295e 100644 --- a/src/relay/relay-handshake-roundtrip.test.ts +++ b/src/relay/relay-handshake-roundtrip.test.ts @@ -76,7 +76,10 @@ describe('handshake round-trip over a real Socket pair', () => { return s } - function startDaemon(version: string): Promise<{ + function startDaemon( + version: string, + endpointCredential?: string + ): Promise<{ accepted: Promise<{ sock: Socket; leftover: Buffer }> }> { return new Promise((resolve) => { @@ -95,6 +98,7 @@ describe('handshake round-trip over a real Socket pair', () => { trackServerSocket(sock) setupDaemonHandshake(sock, { launchVersion: version, + endpointCredential, onAccepted: (s, leftover) => acceptedDeferred.resolve({ sock: s, leftover }) }) }) @@ -120,6 +124,23 @@ describe('handshake round-trip over a real Socket pair', () => { bridgeSock.destroy() }) + it('rejects a same-build socket that lacks the detached endpoint credential', async () => { + const { accepted } = await startDaemon('0.1.0+match', 'secret-credential') + const bridgeSock = connect(sockPath) + await new Promise((resolve) => bridgeSock.once('connect', resolve)) + const closed = new Promise((resolve) => bridgeSock.once('close', () => resolve())) + + runConnectHandshake(bridgeSock, '0.1.0+match', { onAccepted: vi.fn() }, 'wrong-credential') + + await closed + await expect( + Promise.race([ + accepted.then(() => 'accepted'), + new Promise((resolve) => setTimeout(() => resolve('closed'), 20)) + ]) + ).resolves.toBe('closed') + }) + it('preserves leftover bytes on the daemon side when an extra frame is coalesced after the handshake', async () => { // Why: simulate an aggressive client that pipelines a frame immediately // after the handshake. We bypass runConnectHandshake here and write the diff --git a/src/relay/relay-handshake.ts b/src/relay/relay-handshake.ts index e021c392561..bb61e8a08af 100644 --- a/src/relay/relay-handshake.ts +++ b/src/relay/relay-handshake.ts @@ -51,6 +51,7 @@ export type DaemonHandshakeCallbacks = { // leftover: bytes buffered after the handshake frame; caller must feed the dispatcher before attaching the data listener or they're lost. onAccepted: (sock: Socket, leftover: Buffer) => void launchVersion: string + endpointCredential?: string } // Why: read one handshake frame before attaching the dispatcher; version mismatch closes the socket so the bridge exits 42. @@ -61,7 +62,12 @@ export function setupDaemonHandshake(sock: Socket, cb: DaemonHandshakeCallbacks) if (handshakeResolved) { return } - const accepted = handleDaemonHandshakeFrame(sock, frame, cb.launchVersion) + const accepted = handleDaemonHandshakeFrame( + sock, + frame, + cb.launchVersion, + cb.endpointCredential + ) if (accepted) { handshakeResolved = true const leftover = decoder.drain() @@ -94,7 +100,8 @@ export function detachHandshakeListener(sock: Socket): void { function handleDaemonHandshakeFrame( sock: Socket, frame: DecodedFrame, - launchVersion: string + launchVersion: string, + endpointCredential?: string ): boolean { if (frame.type !== MessageType.Handshake) { process.stderr.write( @@ -134,6 +141,14 @@ function handleDaemonHandshakeFrame( sock.end() return false } + if ( + endpointCredential !== undefined && + ('endpointCredential' in msg ? msg.endpointCredential : undefined) !== endpointCredential + ) { + relayLogLine('[relay] Endpoint credential mismatch; closing socket') + sock.destroy() + return false + } process.stderr.write(`[relay] Handshake OK from version=${msg.version}\n`) sock.write(encodeHandshakeFrame({ type: 'orca-relay-handshake-ok', version: launchVersion })) return true @@ -150,7 +165,8 @@ export type ConnectHandshakeCallbacks = { export function runConnectHandshake( sock: Socket, myVersion: string, - cb: ConnectHandshakeCallbacks + cb: ConnectHandshakeCallbacks, + endpointCredential?: string ): void { let handshakeDone = false @@ -212,5 +228,11 @@ export function runConnectHandshake( } }) - sock.write(encodeHandshakeFrame({ type: 'orca-relay-handshake', version: myVersion })) + sock.write( + encodeHandshakeFrame({ + type: 'orca-relay-handshake', + version: myVersion, + ...(endpointCredential ? { endpointCredential } : {}) + }) + ) } diff --git a/src/relay/relay-pty-source-activation.ts b/src/relay/relay-pty-source-activation.ts new file mode 100644 index 00000000000..d4ab4e06726 --- /dev/null +++ b/src/relay/relay-pty-source-activation.ts @@ -0,0 +1,105 @@ +import type { + PtySourceRecoveryCheckpoint, + PtySourceRecoveryRequest, + PtySourceRecoveryResult +} from '../shared/pty-source-recovery-contract' +import type { PtySourceReceivingActivation } from '../shared/pty-source-receiving-activation' +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import type { RequestContext } from './dispatcher' +import type { + RelayPtySourceDeliveryRecord, + RelayPtySourceSendScheduler +} from './relay-pty-source-send-scheduler' +import type { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +export function createPtySourceReceivingActivation( + identity: PtySourceDeliveryIdentity, + checkpointSourceEndSu: number, + recoveryEndSu: number +): PtySourceReceivingActivation { + return Object.freeze({ + status: 'pending', + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration, + ptyIncarnation: identity.ptyIncarnation, + deliveryToken: identity.deliveryToken, + checkpointSourceEndSu, + recoveryEndSu + }) +} + +export function registerPtySourceActivationSettlement(options: { + id: string + record: RelayPtySourceDeliveryRecord + context: RequestContext + deliveries: Map + session: SshPtyConsumerSessionAdapter + sender: RelayPtySourceSendScheduler + onCapacity: (id: string) => void +}): void { + const { id, record, context, deliveries, session, sender, onCapacity } = options + context.onResponseSettled!((result) => { + if (deliveries.get(id) !== record || !record.activating) { + return + } + if (!result.ok) { + if (!record.activationRecoveryRequest) { + session.cancelDelivery(record.identity, 'activation-publication-failed') + deliveries.delete(id) + } + return + } + record.activating = false + sender.completeRecoveryIfReady(record) + sender.pump(record) + onCapacity(id) + }) +} + +export function registerCanceledPtySourceRetirement( + record: RelayPtySourceDeliveryRecord, + context: RequestContext, + deliveries: Map, + onCapacity: (id: string) => void +): void { + record.recoveryCheckpointSourceEndSu = null + record.recoveryEndSu = null + context.onResponseSettled!(() => { + if (deliveries.get(record.identity.id) !== record) { + return + } + deliveries.delete(record.identity.id) + onCapacity(record.identity.id) + }) +} + +export function pendingPtySourceRecoveryResult( + record: RelayPtySourceDeliveryRecord +): PtySourceRecoveryResult { + if (record.recoveryCheckpointSourceEndSu === null || record.recoveryEndSu === null) { + return Object.freeze({ status: 'restoreRequired', reason: 'checkpointUnavailable' }) + } + return Object.freeze({ + status: 'pending', + clientGeneration: record.identity.clientGeneration, + ownerGeneration: record.identity.ownerGeneration, + ptyIncarnation: record.identity.ptyIncarnation, + deliveryToken: record.identity.deliveryToken, + checkpointSourceEndSu: record.recoveryCheckpointSourceEndSu, + recoveryEndSu: record.recoveryEndSu + }) +} + +export function samePtySourceRecoveryRequest( + expected: PtySourceRecoveryCheckpoint, + received: PtySourceRecoveryRequest | undefined +): boolean { + return ( + received?.status === 'checkpoint' && + received.deliveryToken === expected.deliveryToken && + received.clientGeneration === expected.clientGeneration && + received.ownerGeneration === expected.ownerGeneration && + received.ptyIncarnation === expected.ptyIncarnation && + received.acceptedSourceEndSu === expected.acceptedSourceEndSu + ) +} diff --git a/src/relay/relay-pty-source-exit-publication.ts b/src/relay/relay-pty-source-exit-publication.ts new file mode 100644 index 00000000000..76990942fc3 --- /dev/null +++ b/src/relay/relay-pty-source-exit-publication.ts @@ -0,0 +1,73 @@ +import type { RelayDispatcher } from './dispatcher' +import { + onceSinkSettlement, + type RelayPtySourceDeliveryRecord, + type RelayPtySourcePublicationCounters, + type RelayPtySourceSendScheduler +} from './relay-pty-source-send-scheduler' +import type { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +type ExitParams = { id: string; code: number; incarnationId: string } + +export function sealAndPublishPtySourceExit(options: { + params: ExitParams + record: RelayPtySourceDeliveryRecord + deliveries: Map + dispatcher: RelayDispatcher + session: SshPtyConsumerSessionAdapter + sender: RelayPtySourceSendScheduler + counters: RelayPtySourcePublicationCounters + onCapacity: (id: string) => void +}): boolean { + const { params, record, deliveries, dispatcher, session, sender, counters, onCapacity } = options + if (record.restoreRequired) { + const published = dispatcher.tryNotifyPtyExit(params) + if (published && deliveries.get(params.id) === record) { + deliveries.delete(params.id) + } + return published + } + if (!record.sealed) { + session.sealDelivery(record.identity) + record.sealed = true + } + sender.pump(record) + const snapshot = session.sourceDeliverySnapshot(record.identity) + if (snapshot.sentEndSu !== snapshot.receivedEndSu) { + return false + } + if (!record.legacyExitAccepted) { + record.legacyExitAccepted = dispatcher.projectPtyExitToMatchingClients( + (clientId) => session.deliveryMode(clientId) !== 'source-owner', + params + ) + if (!record.legacyExitAccepted) { + return false + } + } + if (record.sourceExitState !== 'idle') { + return true + } + record.sourceExitState = 'pending' + const settle = onceSinkSettlement((result) => { + try { + session.settleExitPublication(record.identity, result) + if (result.ok) { + record.sourceExitState = 'published' + counters.exitCommitted++ + } else { + record.sourceExitState = 'idle' + counters.exitRolledBack++ + } + } finally { + if (result.ok) { + onCapacity(params.id) + } + } + }) + const accepted = dispatcher.tryNotifyPtyExitToClient(record.clientId, params, settle) + if (!accepted && record.sourceExitState === 'pending') { + record.sourceExitState = 'idle' + } + return accepted +} diff --git a/src/relay/relay-pty-source-output.ts b/src/relay/relay-pty-source-output.ts new file mode 100644 index 00000000000..6beefefa6cd --- /dev/null +++ b/src/relay/relay-pty-source-output.ts @@ -0,0 +1,8 @@ +export type RelayPtySourceOutput = { + data: string + rawLength?: number + transformed?: boolean + seq?: number + sourceAccepted?: boolean + sourceSpanId?: string +} diff --git a/src/relay/relay-pty-source-publication.test.ts b/src/relay/relay-pty-source-publication.test.ts new file mode 100644 index 00000000000..3ce4526efa6 --- /dev/null +++ b/src/relay/relay-pty-source-publication.test.ts @@ -0,0 +1,745 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + RelayDispatcher, + type RelayClientSessionIdentity, + type SinkWriteSettlement +} from './dispatcher' +import { encodeJsonRpcFrame, MessageType } from './protocol' +import { RelayPtySourcePublication } from './relay-pty-source-publication' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +const endpointIdentity: RelayClientSessionIdentity = { + principal: 'endpoint-principal', + authenticated: true, + allowSessionOwner: true, + authenticationKind: 'endpoint-credential' +} + +function requestFrame(id: number, method: string, params: Record): Buffer { + return encodeJsonRpcFrame({ jsonrpc: '2.0', id, method, params }, id, 0) +} + +function notification(buffer: Buffer): { method: string; params: Record } | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + const message = JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) + return typeof message.method === 'string' && message.id === undefined ? message : null +} + +function responseResult(buffer: Buffer): Record | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + const message = JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) + return message.id === undefined ? null : (message.result ?? null) +} + +async function flushRequests(): Promise { + await new Promise((resolve) => setImmediate(resolve)) +} + +describe('RelayPtySourcePublication', () => { + let dispatcher: RelayDispatcher | null = null + + afterEach(() => { + dispatcher?.dispose() + dispatcher = null + }) + + async function createHarness( + windowSu = 4, + settleSourceImmediately = true, + highWaterMark?: number + ) { + const writes: Buffer[] = [] + const sourceSettlements: ((result: SinkWriteSettlement) => void)[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + writes.push(Buffer.from(data)) + const frame = notification(data) + if (frame?.method === 'pty.data' || frame?.method === 'pty.exit') { + sourceSettlements.push(onSettled) + if (settleSourceImmediately) { + onSettled({ ok: true }) + } + } else { + onSettled({ ok: true }) + } + return true + }, + { + supportsWriteCallback: true, + ...(highWaterMark + ? { + writableLength: () => 0, + writableHighWaterMark: () => highWaterMark + } + : {}) + }, + endpointIdentity + ) + let publication: RelayPtySourcePublication + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, () => {}) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: windowSu } } + }) + ) + await flushRequests() + const activationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + expect( + publication.activate('pty-1', 'incarnation-1', { + clientId: 1, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => activationSettlements.push(callback) + }) + ).toBe('opened') + activationSettlements[0]({ ok: true }) + return { adapter, publication, sourceSettlements, writes } + } + + it('commits only from writer settlement and resumes exactly after cumulative ACK', async () => { + const harness = await createHarness(4, false) + expect(harness.publication.publish('pty-1', { data: 'abcdefgh' }, false)).toBe(true) + expect(harness.publication.getDebugSnapshot()).toMatchObject({ + outstandingSourceUnits: 0, + sendCommitted: 0 + }) + + harness.sourceSettlements[0]({ ok: true }) + expect(harness.publication.getDebugSnapshot()).toMatchObject({ + outstandingSourceUnits: 4, + sendCommitted: 1 + }) + const first = harness.writes.map(notification).find((frame) => frame?.method === 'pty.data')! + dispatcher!.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: [ + { + id: 'pty-1', + clientGeneration: first.params.clientGeneration, + ownerGeneration: first.params.ownerGeneration, + deliveryToken: first.params.deliveryToken, + creditedEndSu: 4 + } + ] + } + }, + 2, + 0 + ) + ) + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(2) + await flushRequests() + }) + + it('settles a reentrant cumulative ACK only after the matching writer callback', async () => { + const harness = await createHarness(4, false) + expect(harness.publication.publish('pty-1', { data: 'abcdefgh' }, false)).toBe(true) + const first = harness.writes.map(notification).find((frame) => frame?.method === 'pty.data')! + + dispatcher!.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: [ + { + id: 'pty-1', + clientGeneration: first.params.clientGeneration, + ownerGeneration: first.params.ownerGeneration, + deliveryToken: first.params.deliveryToken, + creditedEndSu: 4 + } + ] + } + }, + 2, + 0 + ) + ) + expect(harness.publication.getDebugSnapshot()).toMatchObject({ + outstandingSourceUnits: 0, + sendCommitted: 0 + }) + + harness.sourceSettlements[0]({ ok: true }) + + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(2) + expect(harness.publication.getDebugSnapshot()).toMatchObject({ + outstandingSourceUnits: 0, + sendCommitted: 1 + }) + }) + + it('slices source frames to the encoded HWM-minus-reserve capacity', async () => { + const highWaterMark = 4096 + const harness = await createHarness(10_000, true, highWaterMark) + const payload = '\u0000'.repeat(4000) + + expect(harness.publication.publish('pty-1', { data: payload }, false)).toBe(true) + for (let turn = 0; turn < 4; turn++) { + await flushRequests() + } + + const frames = harness.writes.filter((buffer) => notification(buffer)?.method === 'pty.data') + expect(frames.length).toBeGreaterThan(1) + expect(Math.max(...frames.map((buffer) => buffer.length))).toBeLessThanOrEqual(3072) + expect( + frames + .map(notification) + .map((frame) => frame!.params.data) + .join('') + ).toBe(payload) + }) + + it('keeps mixed legacy and V1 clients on distinct frame authority', async () => { + const harness = await createHarness(8) + const legacyWrites: Buffer[] = [] + dispatcher!.attachClient( + (data, onSettled) => { + legacyWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true } + ) + + harness.publication.publish('pty-1', { data: 'data' }, false) + + const sourceFrame = harness.writes + .map(notification) + .find((frame) => frame?.method === 'pty.data')! + const oldGrant = harness.writes.map(responseResult).find((result) => result?.ownerLease)! + const legacyFrame = legacyWrites + .map(notification) + .find((frame) => frame?.method === 'pty.data')! + expect(sourceFrame.params).toMatchObject({ + data: 'data', + sourceEndSu: 4, + sourceLengthSu: 4 + }) + expect(legacyFrame.params).toEqual({ id: 'pty-1', data: 'data' }) + + dispatcher!.invalidateClient() + const replacementWrites: Buffer[] = [] + const replacementClientId = dispatcher!.attachClient( + (data, onSettled) => { + replacementWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher!.feedClient( + replacementClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: oldGrant.ownerGeneration, + ownerLease: oldGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + harness.publication.publish('pty-1', { data: 'next' }, false) + + expect( + replacementWrites.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(0) + expect( + legacyWrites.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(2) + }) + + it('detaches only a saturated subscriber while the V1 owner stays live', async () => { + const harness = await createHarness(8) + const detached: number[] = [] + const saturatedWrites: Buffer[] = [] + const healthyWrites: Buffer[] = [] + const heldSettlements: ((result: SinkWriteSettlement) => void)[] = [] + dispatcher!.onClientDetached((clientId) => detached.push(clientId)) + const saturatedId = dispatcher!.attachClient( + (data, onSettled) => { + saturatedWrites.push(Buffer.from(data)) + heldSettlements.push(onSettled) + return false + }, + { + supportsWriteCallback: true, + writableLength: () => 128 * 1024, + writableHighWaterMark: () => 4 * 1024 * 1024 + } + ) + const healthyId = dispatcher!.attachClient( + (data, onSettled) => { + healthyWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true } + ) + const saturatedPayload = 's'.repeat(128 * 1024) + let admitted = 0 + while ( + dispatcher!.tryNotifyPtyDataToClient( + saturatedId, + { id: 'saturated', data: saturatedPayload }, + () => {} + ) + ) { + admitted++ + } + + expect(admitted).toBeGreaterThan(0) + expect(admitted).toBeLessThan(20) + expect(harness.publication.publish('pty-1', { data: saturatedPayload }, false)).toBe(true) + + expect(detached).toEqual([saturatedId]) + expect(detached).not.toContain(healthyId) + expect(saturatedWrites).toHaveLength(1) + expect(heldSettlements).toHaveLength(1) + expect( + healthyWrites.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + expect(harness.publication.getDebugSnapshot()).toMatchObject({ sendCommitted: 1 }) + }) + + it('keeps an early ACK retryable when its source write fails', async () => { + const harness = await createHarness(4, false) + harness.publication.publish('pty-1', { data: 'abcdefgh' }, false) + const first = harness.writes.map(notification).find((frame) => frame?.method === 'pty.data')! + dispatcher!.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: [ + { + id: 'pty-1', + clientGeneration: first.params.clientGeneration, + ownerGeneration: first.params.ownerGeneration, + deliveryToken: first.params.deliveryToken, + creditedEndSu: 4 + } + ] + } + }, + 2, + 0 + ) + ) + harness.sourceSettlements[0]({ ok: false, error: new Error('socket write failed') }) + + expect(harness.publication.getDebugSnapshot()).toMatchObject({ + outstandingSourceUnits: 0, + sendCommitted: 0, + sendRolledBack: 1 + }) + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + expect(harness.adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1, sourceSu: 8 }) + }) + + it('fences idle publication and pumping before the wait continuation', async () => { + const harness = await createHarness(4) + harness.publication.publish('pty-1', { data: 'abcdefgh' }, false) + const firstData = harness.writes + .map(notification) + .find((frame) => frame?.method === 'pty.data')! + const fence = harness.publication.waitForPendingSend('pty-1') + + expect(harness.publication.publish('pty-1', { data: 'ijkl' }, false)).toBe(false) + dispatcher!.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: [ + { + id: 'pty-1', + clientGeneration: firstData.params.clientGeneration, + ownerGeneration: firstData.params.ownerGeneration, + deliveryToken: firstData.params.deliveryToken, + creditedEndSu: 4 + } + ] + } + }, + 2, + 0 + ) + ) + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + expect(await fence).toBe(true) + + expect( + harness.publication.activate('pty-1', 'incarnation-1', { + clientId: 1, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: () => {} + }) + ).toBe('existing') + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(2) + await flushRequests() + }) + + it('does not pump an old-token suffix after a successful fenced settlement', async () => { + const harness = await createHarness(4, false) + harness.publication.publish('pty-1', { data: 'abcdefgh' }, false) + const oldData = harness.writes.map(notification).find((frame) => frame?.method === 'pty.data')! + const oldGrant = harness.writes.map(responseResult).find((result) => result?.ownerLease)! + const fence = harness.publication.waitForPendingSend('pty-1') + + harness.sourceSettlements[0]({ ok: true }) + expect(await fence).toBe(true) + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + dispatcher!.invalidateClient() + + const recoveredWrites: Buffer[] = [] + const recoveredClientId = dispatcher!.attachClient( + (data, onSettled) => { + recoveredWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher!.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: oldGrant.ownerGeneration, + ownerLease: oldGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const activationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + expect( + harness.publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => activationSettlements.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(oldData.params.deliveryToken), + clientGeneration: Number(oldData.params.clientGeneration), + ownerGeneration: Number(oldData.params.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 4 + } + ) + ).toMatchObject({ status: 'pending', checkpointSourceEndSu: 4, recoveryEndSu: 8 }) + activationSettlements[0]({ ok: true }) + + const replacementData = recoveredWrites + .map(notification) + .find((frame) => frame?.method === 'pty.data')! + expect(replacementData.params).toMatchObject({ data: 'efgh', sourceEndSu: 8 }) + expect(replacementData.params.deliveryToken).not.toBe(oldData.params.deliveryToken) + expect( + harness.writes.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(1) + }) + + it('recovers a receiver-accepted frame after detach rolls back its send', async () => { + const harness = await createHarness(4, false) + harness.publication.publish('pty-1', { data: 'data' }, false) + const oldData = harness.writes.map(notification).find((frame) => frame?.method === 'pty.data')! + const oldGrant = harness.writes.map(responseResult).find((result) => result?.ownerLease)! + let fenceSettled = false + const fence = harness.publication.waitForPendingSend('pty-1').then((result) => { + fenceSettled = true + return result + }) + await Promise.resolve() + expect(fenceSettled).toBe(false) + + dispatcher!.invalidateClient() + expect(await fence).toBe(true) + expect(harness.publication.getDebugSnapshot()).toMatchObject({ + sendCommitted: 0, + sendRolledBack: 1 + }) + + const recoveredWrites: Buffer[] = [] + const recoveredClientId = dispatcher!.attachClient( + (data, onSettled) => { + recoveredWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher!.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: oldGrant.ownerGeneration, + ownerLease: oldGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const activationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const activation = harness.publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => activationSettlements.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(oldData.params.deliveryToken), + clientGeneration: Number(oldData.params.clientGeneration), + ownerGeneration: Number(oldData.params.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 4 + } + ) + expect(activation).toMatchObject({ + status: 'pending', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }) + activationSettlements[0]({ ok: true }) + expect( + recoveredWrites.map(notification).filter((frame) => frame?.method === 'pty.data') + ).toHaveLength(0) + expect( + recoveredWrites.map(notification).find((frame) => frame?.method === 'pty.recoveryComplete') + ?.params + ).toMatchObject({ checkpointSourceEndSu: 4, recoveryEndSu: 4 }) + const beforeLateSettlement = harness.publication.getDebugSnapshot() + + harness.sourceSettlements[0]({ ok: true }) + + expect(harness.publication.getDebugSnapshot()).toEqual(beforeLateSettlement) + }) + + it('publishes exit only after preceding source data settles', async () => { + const harness = await createHarness(4, false) + harness.publication.publish('pty-1', { data: 'data' }, false) + expect( + harness.publication.sealAndPublishExit({ + id: 'pty-1', + code: 0, + incarnationId: 'incarnation-1' + }) + ).toBe(false) + + harness.sourceSettlements[0]({ ok: true }) + expect( + harness.publication.sealAndPublishExit({ + id: 'pty-1', + code: 0, + incarnationId: 'incarnation-1' + }) + ).toBe(true) + expect( + harness.writes + .map(notification) + .filter((frame): frame is NonNullable => frame !== null) + .map((frame) => frame.method) + ).toEqual(['pty.data', 'pty.exit']) + }) + + it('settles the recovery fence before sending buffered live output', async () => { + const harness = await createHarness(4) + harness.publication.publish('pty-1', { data: 'abcdefgh' }, false) + const firstData = harness.writes + .map(notification) + .find((frame) => frame?.method === 'pty.data')! + const firstGrant = harness.writes.map(responseResult).find((result) => result?.ownerLease)! + dispatcher!.invalidateClient() + + const recoveredWrites: Buffer[] = [] + const recoverySettlements: ((result: SinkWriteSettlement) => void)[] = [] + const completionSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const recoveredClientId = dispatcher!.attachClient( + (data, onSettled) => { + recoveredWrites.push(Buffer.from(data)) + const method = notification(data)?.method + if (method === 'pty.data') { + recoverySettlements.push(onSettled) + } else if (method === 'pty.recoveryComplete') { + completionSettlements.push(onSettled) + } else { + onSettled({ ok: true }) + } + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher!.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: firstGrant.ownerGeneration, + ownerLease: firstGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const activationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const activation = harness.publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => activationSettlements.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(firstData.params.deliveryToken), + clientGeneration: Number(firstData.params.clientGeneration), + ownerGeneration: Number(firstData.params.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 4 + } + ) + expect(activation).toMatchObject({ + status: 'pending', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }) + activationSettlements[0]({ ok: true }) + expect(harness.publication.publish('pty-1', { data: 'ijkl' }, false)).toBe(false) + + expect( + recoveredWrites.map(notification).filter((frame) => frame?.method === 'pty.recoveryComplete') + ).toHaveLength(0) + const recoveredData = recoveredWrites + .map(notification) + .find((frame) => frame?.method === 'pty.data')! + harness.adapter.appendSource( + { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: Number(recoveredData.params.clientGeneration), + ownerGeneration: Number(recoveredData.params.ownerGeneration), + ptyIncarnation: String(recoveredData.params.ptyIncarnation), + deliveryToken: String(recoveredData.params.deliveryToken) + }, + { + spanId: 'buffered-live', + data: 'ijkl', + displayStart: 8, + displayEnd: 12, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + } + ) + recoverySettlements[0]({ ok: true }) + expect( + recoveredWrites + .map(notification) + .filter((frame): frame is NonNullable => frame !== null) + .map((frame) => frame.method) + ).toEqual(['pty.deliveryCanceled', 'pty.data', 'pty.recoveryComplete']) + dispatcher!.feedClient( + recoveredClientId, + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: [ + { + id: 'pty-1', + clientGeneration: recoveredData.params.clientGeneration, + ownerGeneration: recoveredData.params.ownerGeneration, + deliveryToken: recoveredData.params.deliveryToken, + creditedEndSu: 8 + } + ] + } + }, + 3, + 0 + ) + ) + expect(harness.publication.publish('pty-1', { data: 'ijkl' }, false)).toBe(false) + expect( + recoveredWrites + .map(notification) + .filter((frame): frame is NonNullable => frame !== null) + .map((frame) => frame.method) + ).toEqual(['pty.deliveryCanceled', 'pty.data', 'pty.recoveryComplete']) + + completionSettlements[0]({ ok: true }) + expect( + recoveredWrites + .map(notification) + .filter((frame): frame is NonNullable => frame !== null) + .map((frame) => frame.method) + ).toEqual(['pty.deliveryCanceled', 'pty.data', 'pty.recoveryComplete', 'pty.data']) + expect(recoveredWrites.map(notification).at(-1)?.params).toMatchObject({ + data: 'ijkl', + sourceEndSu: 12 + }) + await flushRequests() + }) +}) diff --git a/src/relay/relay-pty-source-publication.ts b/src/relay/relay-pty-source-publication.ts new file mode 100644 index 00000000000..10ef73b0ce6 --- /dev/null +++ b/src/relay/relay-pty-source-publication.ts @@ -0,0 +1,316 @@ +import { randomUUID } from 'node:crypto' +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import type { + PtySourceRecoveryRequest, + PtySourceRecoveryResult +} from '../shared/pty-source-recovery-contract' +import type { PtySourceReceivingActivation } from '../shared/pty-source-receiving-activation' +import { + createPtySourceReceivingActivation, + pendingPtySourceRecoveryResult, + registerCanceledPtySourceRetirement, + registerPtySourceActivationSettlement, + samePtySourceRecoveryRequest +} from './relay-pty-source-activation' +import { + RelayPtySourceSendScheduler, + type RelayPtySourceDeliveryRecord, + type RelayPtySourcePublicationCounters +} from './relay-pty-source-send-scheduler' +import { sealAndPublishPtySourceExit } from './relay-pty-source-exit-publication' +import type { RelayDispatcher, RequestContext } from './dispatcher' +import type { RelayPtySourceOutput } from './relay-pty-source-output' +import type { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +export class RelayPtySourcePublication { + private readonly deliveries = new Map() + private readonly counters: RelayPtySourcePublicationCounters = { + opened: 0, + rotated: 0, + appendDenied: 0, + sendCommitted: 0, + sendRolledBack: 0, + exitCommitted: 0, + exitRolledBack: 0 + } + + constructor( + private readonly dispatcher: RelayDispatcher, + private readonly session: SshPtyConsumerSessionAdapter, + private readonly onCapacity: (id: string) => void + ) { + this.sender = new RelayPtySourceSendScheduler( + dispatcher, + session, + this.deliveries, + this.counters, + onCapacity + ) + } + + private readonly sender: RelayPtySourceSendScheduler + + activate( + id: string, + ptyIncarnation: string, + context: RequestContext | undefined, + recovery?: PtySourceRecoveryRequest + ): false | 'opened' | 'rotated' | 'existing' | PtySourceRecoveryResult { + if (!context?.onResponseSettled) { + this.sender.releaseRotationFence(this.deliveries.get(id)) + return false + } + const mode = this.session.deliveryMode(context.clientId) + const current = this.deliveries.get(id) + if (mode === 'subscriber') { + this.sender.releaseRotationFence(current) + return false + } + if (mode === 'legacy-owner') { + if (current) { + this.session.cancelDelivery(current.identity, 'source-credit-disabled') + this.deliveries.delete(id) + this.onCapacity(id) + } + return false + } + if (current?.clientId === context.clientId) { + this.sender.releaseRotationFence(current) + if (current.activating && current.activationRecoveryRequest) { + if (!samePtySourceRecoveryRequest(current.activationRecoveryRequest, recovery)) { + return this.publishRestoreRequired(id, context, 'checkpointUnavailable') + } + this.registerActivationSettlement(id, current, context) + return pendingPtySourceRecoveryResult(current) + } + return 'existing' + } + let identity: PtySourceDeliveryIdentity | null = null + let displayEnd = 0 + let recoveryCheckpointSourceEndSu: number | null = null + let recoveryEndSu: number | null = null + let recoveryWasSealed = false + if (!current && recovery) { + return this.publishRestoreRequired(id, context, 'deliveryUnavailable') + } + if (current) { + try { + const snapshot = this.session.sourceDeliverySnapshot(current.identity) + if ( + snapshot.state === 'closed' || + snapshot.state === 'closing' || + recovery?.status !== 'checkpoint' || + recovery.deliveryToken !== current.identity.deliveryToken || + recovery.clientGeneration !== current.identity.clientGeneration || + recovery.ownerGeneration !== current.identity.ownerGeneration || + recovery.ptyIncarnation !== current.identity.ptyIncarnation + ) { + return this.requireRestore(id, current, context, 'checkpointUnavailable') + } + const rotation = this.session.rotateDelivery( + current.identity, + context.clientId, + recovery.acceptedSourceEndSu + ) + identity = rotation.identity + displayEnd = current.displayEnd + recoveryCheckpointSourceEndSu = recovery.acceptedSourceEndSu + recoveryEndSu = snapshot.receivedEndSu + recoveryWasSealed = snapshot.state === 'sealed-unsettled' + this.counters.rotated++ + } catch (error) { + return this.requireRestore( + id, + current, + context, + error instanceof Error ? error.message : 'invalidCheckpoint' + ) + } + } + identity ??= this.session.openDelivery(context.clientId, id, ptyIncarnation) + if (!identity) { + return false + } + if (!current || identity !== current.identity) { + this.counters.opened++ + } + const activationSnapshot = this.session.sourceDeliverySnapshot(identity) + const activationCheckpointSourceEndSu = + recoveryCheckpointSourceEndSu ?? activationSnapshot.sentEndSu + const activationRecoveryEndSu = recoveryEndSu ?? activationSnapshot.receivedEndSu + const record: RelayPtySourceDeliveryRecord = { + clientId: context.clientId, + identity, + sourceActivation: createPtySourceReceivingActivation( + identity, + activationCheckpointSourceEndSu, + activationRecoveryEndSu + ), + displayEnd, + activating: true, + activationRecoveryRequest: + recovery?.status === 'checkpoint' ? Object.freeze({ ...recovery }) : null, + sealed: recoveryWasSealed, + legacyExitAccepted: false, + sourceExitState: 'idle', + sending: false, + turnFrames: 0, + turnSourceSu: 0, + turnScheduled: false, + sendWaiters: new Set(), + recoveryCheckpointSourceEndSu, + recoveryEndSu, + recoveryCompletionPending: false, + restoreRequired: false, + rotationPending: false + } + this.deliveries.set(id, record) + this.registerActivationSettlement(id, record, context) + if (recoveryEndSu !== null && recoveryCheckpointSourceEndSu !== null) { + return pendingPtySourceRecoveryResult(record) + } + return current ? 'rotated' : 'opened' + } + + accepts = (id: string): boolean => this.deliveries.has(id) + + receivingActivation(id: string, clientId: number): PtySourceReceivingActivation | undefined { + const record = this.deliveries.get(id) + return record?.clientId === clientId && !record.restoreRequired + ? record.sourceActivation + : undefined + } + + waitForPendingSend = (id: string, timeoutMs = 5_000): Promise => + this.sender.waitForPendingSend(id, timeoutMs) + + publish(id: string, output: RelayPtySourceOutput, interactive: boolean): boolean { + const record = this.deliveries.get(id) + if ( + !record || + record.sealed || + record.recoveryEndSu !== null || + record.restoreRequired || + record.rotationPending + ) { + return false + } + if (!output.sourceAccepted) { + const rawLength = output.rawLength ?? output.data.length + output.sourceSpanId ??= randomUUID() + try { + this.session.appendSource(record.identity, { + spanId: output.sourceSpanId, + data: output.data, + displayStart: record.displayEnd, + displayEnd: record.displayEnd + output.data.length, + splittable: output.transformed !== true, + transform: { + transformed: output.transformed === true, + rawLengthSu: rawLength, + scalarSafe: output.transformed !== true + } + }) + } catch { + this.counters.appendDenied++ + return false + } + output.sourceAccepted = true + record.displayEnd += output.data.length + } + if ( + !this.dispatcher.projectPtyDataToMatchingClients( + (clientId) => this.session.deliveryMode(clientId) !== 'source-owner', + { + id, + data: output.data, + ...(output.seq === undefined ? {} : { seq: output.seq }), + ...(output.rawLength === undefined ? {} : { rawLength: output.rawLength }), + ...(output.transformed ? { transformed: true } : {}) + }, + { interactive } + ) + ) { + return false + } + this.sender.pump(record) + return true + } + + sealAndPublishExit(params: { id: string; code: number; incarnationId: string }): boolean { + const record = this.deliveries.get(params.id) + if (!record) { + return false + } + return sealAndPublishPtySourceExit({ + params, + record, + deliveries: this.deliveries, + dispatcher: this.dispatcher, + session: this.session, + sender: this.sender, + counters: this.counters, + onCapacity: this.onCapacity + }) + } + + onCreditAvailable = (id: string): void => this.sender.onCreditAvailable(id) + + exitPublicationSettled(id: string): boolean { + const record = this.deliveries.get(id) + if (!record || record.sourceExitState !== 'published') { + return false + } + this.sender.pruneClosed(id, record) + return true + } + + getDebugSnapshot = () => this.sender.getDebugSnapshot() + + dispose = (): void => this.sender.dispose() + + private registerActivationSettlement( + id: string, + record: RelayPtySourceDeliveryRecord, + context: RequestContext + ): void { + registerPtySourceActivationSettlement({ + id, + record, + context, + deliveries: this.deliveries, + session: this.session, + sender: this.sender, + onCapacity: this.onCapacity + }) + } + + private requireRestore( + id: string, + current: RelayPtySourceDeliveryRecord, + context: RequestContext, + reason: string + ): Readonly<{ status: 'restoreRequired'; reason: string }> { + this.session.cancelDelivery(current.identity, `recovery-${reason}`) + current.restoreRequired = true + current.activating = false + this.sender.wakeSendWaiters(current) + registerCanceledPtySourceRetirement(current, context, this.deliveries, this.onCapacity) + return this.publishRestoreRequired(id, context, reason) + } + + private publishRestoreRequired( + id: string, + context: RequestContext, + reason: string + ): Readonly<{ status: 'restoreRequired'; reason: string }> { + const result = Object.freeze({ status: 'restoreRequired' as const, reason }) + context.onResponseSettled?.((settlement) => { + if (settlement.ok) { + this.dispatcher.notifyClient(context.clientId, 'pty.restoreRequired', { id, reason }) + } + }) + this.onCapacity(id) + return result + } +} diff --git a/src/relay/relay-pty-source-recovery-completion.test.ts b/src/relay/relay-pty-source-recovery-completion.test.ts new file mode 100644 index 00000000000..ba5f8d6787f --- /dev/null +++ b/src/relay/relay-pty-source-recovery-completion.test.ts @@ -0,0 +1,117 @@ +import { expect, it } from 'vitest' +import type { RelayDispatcher, SinkWriteSettlement } from './dispatcher' +import type { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' +import { + RelayPtySourceSendScheduler, + type RelayPtySourceDeliveryRecord +} from './relay-pty-source-send-scheduler' + +it('retries an unadmitted recovery completion once capacity returns', () => { + let capacityListener = () => {} + let listenerRemoved = false + let admissions = 0 + const completionSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const dispatcher = { + onLegacyPtyCapacity(listener: () => void) { + capacityListener = listener + return () => { + listenerRemoved = true + } + }, + tryNotifyClient( + _clientId: number, + _method: string, + _params: Record, + onSettled: (result: SinkWriteSettlement) => void + ) { + admissions++ + if (admissions === 1) { + return false + } + completionSettlements.push(onSettled) + return true + }, + producerDataBudget: () => 1024, + tryNotifyPtyDataToClient: () => false + } as unknown as RelayDispatcher + const session = { + sourceDeliverySnapshot: () => ({ sentEndSu: 4, state: 'active' }), + reserveSourceSend: () => null, + cancelDelivery: () => {} + } as unknown as SshPtyConsumerSessionAdapter + const record: RelayPtySourceDeliveryRecord = { + clientId: 1, + identity: { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1' + }, + sourceActivation: { + status: 'pending', + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + checkpointSourceEndSu: 0, + recoveryEndSu: 4 + }, + displayEnd: 4, + activating: false, + activationRecoveryRequest: null, + sealed: false, + legacyExitAccepted: false, + sourceExitState: 'idle', + sending: false, + turnFrames: 0, + turnSourceSu: 0, + turnScheduled: false, + sendWaiters: new Set(), + recoveryCheckpointSourceEndSu: 0, + recoveryEndSu: 4, + recoveryCompletionPending: false, + restoreRequired: false, + rotationPending: false + } + const deliveries = new Map([['pty-1', record]]) + let capacityCalls = 0 + const scheduler = new RelayPtySourceSendScheduler( + dispatcher, + session, + deliveries, + { + opened: 0, + rotated: 0, + appendDenied: 0, + sendCommitted: 0, + sendRolledBack: 0, + exitCommitted: 0, + exitRolledBack: 0 + }, + () => { + capacityCalls++ + } + ) + + scheduler.completeRecoveryIfReady(record) + expect(admissions).toBe(1) + expect(record.recoveryCompletionPending).toBe(false) + expect(record.recoveryEndSu).toBe(4) + + capacityListener() + capacityListener() + expect(admissions).toBe(2) + expect(completionSettlements).toHaveLength(1) + expect(record.recoveryCompletionPending).toBe(true) + + completionSettlements[0]({ ok: true }) + capacityListener() + expect(admissions).toBe(2) + expect(record.recoveryEndSu).toBeNull() + expect(capacityCalls).toBe(1) + + scheduler.dispose() + expect(listenerRemoved).toBe(true) +}) diff --git a/src/relay/relay-pty-source-recovery-completion.ts b/src/relay/relay-pty-source-recovery-completion.ts new file mode 100644 index 00000000000..bc4c79e1be6 --- /dev/null +++ b/src/relay/relay-pty-source-recovery-completion.ts @@ -0,0 +1,68 @@ +import type { RelayDispatcher, SinkWriteSettlement } from './dispatcher' +import type { RelayPtySourceDeliveryRecord } from './relay-pty-source-send-scheduler' +import type { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +function onceSettlement( + callback: (result: SinkWriteSettlement) => void +): (result: SinkWriteSettlement) => void { + let settled = false + return (result) => { + if (settled) { + return + } + settled = true + callback(result) + } +} + +export function completePtySourceRecovery(options: { + record: RelayPtySourceDeliveryRecord + deliveries: Map + dispatcher: RelayDispatcher + session: SshPtyConsumerSessionAdapter + onCompleted: (id: string) => void +}): void { + const { record, deliveries, dispatcher, session, onCompleted } = options + const recoveryEndSu = record.recoveryEndSu + const checkpointSourceEndSu = record.recoveryCheckpointSourceEndSu + if ( + record.activating || + record.sending || + record.recoveryCompletionPending || + recoveryEndSu === null || + checkpointSourceEndSu === null || + session.sourceDeliverySnapshot(record.identity).sentEndSu < recoveryEndSu + ) { + return + } + record.recoveryCompletionPending = true + const settle = onceSettlement((result) => { + if (deliveries.get(record.identity.id) !== record) { + return + } + record.recoveryCompletionPending = false + if (!result.ok) { + return + } + record.recoveryEndSu = null + record.recoveryCheckpointSourceEndSu = null + onCompleted(record.identity.id) + }) + const accepted = dispatcher.tryNotifyClient( + record.clientId, + 'pty.recoveryComplete', + { + id: record.identity.id, + clientGeneration: record.identity.clientGeneration, + ownerGeneration: record.identity.ownerGeneration, + ptyIncarnation: record.identity.ptyIncarnation, + deliveryToken: record.identity.deliveryToken, + checkpointSourceEndSu, + recoveryEndSu + }, + settle + ) + if (!accepted) { + settle({ ok: false, error: new Error('PTY recovery completion was not admitted') }) + } +} diff --git a/src/relay/relay-pty-source-recovery-interleavings.test.ts b/src/relay/relay-pty-source-recovery-interleavings.test.ts new file mode 100644 index 00000000000..0c35b25c1a7 --- /dev/null +++ b/src/relay/relay-pty-source-recovery-interleavings.test.ts @@ -0,0 +1,518 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import type { PtySourceRecoveryRequest } from '../shared/pty-source-recovery-contract' +import { + RelayDispatcher, + type RelayClientSessionIdentity, + type SinkWriteSettlement +} from './dispatcher' +import { encodeJsonRpcFrame, MessageType } from './protocol' +import { RelayPtySourceCreditLedger } from './pty-source-credit-ledger' +import { RelayPtySourcePublication } from './relay-pty-source-publication' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +const endpointIdentity: RelayClientSessionIdentity = { + principal: 'endpoint-principal', + authenticated: true, + allowSessionOwner: true, + authenticationKind: 'endpoint-credential' +} + +function deliveryIdentity( + deliveryToken: string, + overrides: Partial = {} +): PtySourceDeliveryIdentity { + return { + id: 'pty-1', + providerGeneration: 1, + clientGeneration: 1, + ownerGeneration: 1, + ptyIncarnation: 'incarnation-1', + deliveryToken, + ...overrides + } +} + +function requestFrame(id: number, method: string, params: Record): Buffer { + return encodeJsonRpcFrame({ jsonrpc: '2.0', id, method, params }, id, 0) +} + +function message(buffer: Buffer): Record | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + return JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) +} + +function responseResult( + writes: readonly Buffer[], + id: number +): Record | undefined { + const response = writes.map(message).find((entry) => entry?.id === id) + return response?.result as Record | undefined +} + +async function flushRequests(): Promise { + await new Promise((resolve) => setImmediate(resolve)) +} + +describe('relay PTY source recovery interleavings', () => { + let dispatcher: RelayDispatcher | null = null + + afterEach(() => { + dispatcher?.dispose() + dispatcher = null + }) + + it('retains failed exit delivery for a late ACK and exact sealed recovery', () => { + const ledger = new RelayPtySourceCreditLedger() + const oldIdentity = deliveryIdentity('old-token') + const replacement = deliveryIdentity('replacement-token', { + clientGeneration: 2, + ownerGeneration: 2 + }) + ledger.open(oldIdentity, 4) + ledger.append(oldIdentity, { + spanId: 'tail', + data: 'tail', + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + }) + const send = ledger.reserveNextSend(oldIdentity)! + ledger.commitSend(send) + ledger.seal(oldIdentity) + + ledger.settleExitPublication(oldIdentity, { + ok: false, + error: new Error('socket closed before exit publication') + }) + ledger.acknowledge(oldIdentity, { + id: oldIdentity.id, + clientGeneration: oldIdentity.clientGeneration, + ownerGeneration: oldIdentity.ownerGeneration, + deliveryToken: oldIdentity.deliveryToken, + creditedEndSu: 4 + }) + const rotation = ledger.rotate(oldIdentity, replacement, 4, 4) + + expect(rotation.recovery).toEqual([]) + expect(ledger.snapshot(replacement)).toMatchObject({ + state: 'sealed-unsettled', + creditedEndSu: 4, + receivedEndSu: 4, + exitPublished: false + }) + expect(ledger.cancel(replacement, 'client-request')).toMatchObject({ + deliveryToken: 'replacement-token', + remainingStartSu: 4, + remainingEndSu: 4 + }) + expect(ledger.snapshot(replacement).state).toBe('closed') + }) + + it('retries a failed exit publication after exact owner recovery', async () => { + const primaryWrites: Buffer[] = [] + let exitSettlement: ((result: SinkWriteSettlement) => void) | undefined + dispatcher = new RelayDispatcher( + (data, settle) => { + primaryWrites.push(Buffer.from(data)) + if (message(data)?.method === 'pty.exit') { + exitSettlement = settle + return true + } + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + let publication: RelayPtySourcePublication + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, () => {}) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const activationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + publication.activate('pty-1', 'incarnation-1', { + clientId: 1, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => activationSettlements.push(callback) + }) + activationSettlements[0]({ ok: true }) + publication.publish('pty-1', { data: 'tail' }, false) + const oldData = primaryWrites.map(message).find((entry) => entry?.method === 'pty.data')! + .params as Record + const oldGrant = responseResult(primaryWrites, 1)! + + expect( + publication.sealAndPublishExit({ + id: 'pty-1', + code: 0, + incarnationId: 'incarnation-1' + }) + ).toBe(true) + expect(publication.exitPublicationSettled('pty-1')).toBe(false) + exitSettlement!({ ok: false, error: new Error('socket closed') }) + expect(publication.exitPublicationSettled('pty-1')).toBe(false) + expect(publication.getDebugSnapshot()).toMatchObject({ + sealedUnsettled: 1, + exitCommitted: 0, + exitRolledBack: 1 + }) + expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1, sourceSu: 4 }) + + const recoveredWrites: Buffer[] = [] + const recoveredClientId = dispatcher.attachClient( + (data, settle) => { + recoveredWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: oldGrant.ownerGeneration, + ownerLease: oldGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const recoveredActivationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const recovered = publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => recoveredActivationSettlements.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(oldData.deliveryToken), + clientGeneration: Number(oldData.clientGeneration), + ownerGeneration: Number(oldData.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 4 + } + ) + recoveredActivationSettlements[0]({ ok: true }) + + expect(recovered).toMatchObject({ + status: 'pending', + checkpointSourceEndSu: 4, + recoveryEndSu: 4 + }) + expect( + publication.sealAndPublishExit({ + id: 'pty-1', + code: 0, + incarnationId: 'incarnation-1' + }) + ).toBe(true) + expect(publication.exitPublicationSettled('pty-1')).toBe(true) + expect( + recoveredWrites.map(message).filter((entry) => entry?.method === 'pty.exit') + ).toHaveLength(1) + }) + + it('keeps failed recovery activation private and retryable until exact response settles', async () => { + const primaryWrites: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, settle) => { + primaryWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + let publication: RelayPtySourcePublication + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, () => {}) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const activationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + expect( + publication.activate('pty-1', 'incarnation-1', { + clientId: 1, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => activationSettlements.push(callback) + }) + ).toBe('opened') + activationSettlements[0]({ ok: true }) + expect(publication.publish('pty-1', { data: 'abcdefgh' }, false)).toBe(true) + const oldData = primaryWrites.map(message).find((entry) => entry?.method === 'pty.data')! + .params as Record + const oldGrant = responseResult(primaryWrites, 1)! + dispatcher.invalidateClient() + + const recoveredWrites: Buffer[] = [] + const recoveredClientId = dispatcher.attachClient( + (data, settle) => { + recoveredWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: oldGrant.ownerGeneration, + ownerLease: oldGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const recoveryRequest: PtySourceRecoveryRequest = { + status: 'checkpoint', + deliveryToken: String(oldData.deliveryToken), + clientGeneration: Number(oldData.clientGeneration), + ownerGeneration: Number(oldData.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 4 + } + const firstActivationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const firstRecovery = publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => firstActivationSettlements.push(callback) + }, + recoveryRequest + ) + expect(firstRecovery).toMatchObject({ + status: 'pending', + checkpointSourceEndSu: 4, + recoveryEndSu: 8 + }) + + firstActivationSettlements[0]({ ok: false, error: new Error('response publication failed') }) + expect(publication.getDebugSnapshot()).toMatchObject({ active: 0, activating: 1 }) + const writesBeforeRetry = recoveredWrites.length + publication.onCreditAvailable('pty-1') + expect(recoveredWrites).toHaveLength(writesBeforeRetry) + + const retryActivationSettlements: ((result: SinkWriteSettlement) => void)[] = [] + const retriedRecovery = publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => retryActivationSettlements.push(callback) + }, + recoveryRequest + ) + + expect(retriedRecovery).toEqual(firstRecovery) + retryActivationSettlements[0]({ ok: true }) + expect(publication.getDebugSnapshot()).toMatchObject({ active: 1, activating: 0 }) + }) + + it('rolls back a failed recovery fence and republishes it for the next exact owner', async () => { + const primaryWrites: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, settle) => { + primaryWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + let publication: RelayPtySourcePublication + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, () => {}) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const firstActivation: ((result: SinkWriteSettlement) => void)[] = [] + publication.activate('pty-1', 'incarnation-1', { + clientId: 1, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => firstActivation.push(callback) + }) + firstActivation[0]({ ok: true }) + publication.publish('pty-1', { data: 'abcdefgh' }, false) + const firstData = primaryWrites.map(message).find((entry) => entry?.method === 'pty.data')! + .params as Record + const firstGrant = responseResult(primaryWrites, 1)! + dispatcher.invalidateClient() + + const recoveredWrites: Buffer[] = [] + let recoveryDataSettlement: ((result: SinkWriteSettlement) => void) | undefined + let failedCompletionSettlement: ((result: SinkWriteSettlement) => void) | undefined + const recoveredClientId = dispatcher.attachClient( + (data, settle) => { + recoveredWrites.push(Buffer.from(data)) + const method = message(data)?.method + if (method === 'pty.data') { + recoveryDataSettlement = settle + } else if (method === 'pty.recoveryComplete') { + failedCompletionSettlement = settle + } else { + settle({ ok: true }) + } + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: firstGrant.ownerGeneration, + ownerLease: firstGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const recoveredGrant = responseResult(recoveredWrites, 2)! + const recoveredActivation: ((result: SinkWriteSettlement) => void)[] = [] + publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => recoveredActivation.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(firstData.deliveryToken), + clientGeneration: Number(firstData.clientGeneration), + ownerGeneration: Number(firstData.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 4 + } + ) + recoveredActivation[0]({ ok: true }) + recoveryDataSettlement!({ ok: true }) + const recoveredData = recoveredWrites + .map(message) + .find((entry) => entry?.method === 'pty.data')!.params as Record + expect(publication.publish('pty-1', { data: 'live' }, false)).toBe(false) + + failedCompletionSettlement!({ ok: false, error: new Error('completion write failed') }) + + const replacementWrites: Buffer[] = [] + let replacementCompletionSettlement: ((result: SinkWriteSettlement) => void) | undefined + const replacementClientId = dispatcher.attachClient( + (data, settle) => { + replacementWrites.push(Buffer.from(data)) + if (message(data)?.method === 'pty.recoveryComplete') { + replacementCompletionSettlement = settle + } else { + settle({ ok: true }) + } + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + replacementClientId, + requestFrame(3, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: recoveredGrant.ownerGeneration, + ownerLease: recoveredGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const replacementActivation: ((result: SinkWriteSettlement) => void)[] = [] + expect( + publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: replacementClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => replacementActivation.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(recoveredData.deliveryToken), + clientGeneration: Number(recoveredData.clientGeneration), + ownerGeneration: Number(recoveredData.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 8 + } + ) + ).toMatchObject({ status: 'pending', checkpointSourceEndSu: 8, recoveryEndSu: 8 }) + replacementActivation[0]({ ok: true }) + expect(publication.publish('pty-1', { data: 'live' }, false)).toBe(false) + + replacementCompletionSettlement!({ ok: true }) + expect(publication.publish('pty-1', { data: 'live' }, false)).toBe(true) + expect( + replacementWrites.map(message).filter((entry) => entry?.method === 'pty.recoveryComplete') + ).toHaveLength(1) + expect( + replacementWrites.map(message).filter((entry) => entry?.method === 'pty.data') + ).toHaveLength(1) + expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1, sourceSu: 4 }) + }) +}) diff --git a/src/relay/relay-pty-source-restore-retry.test.ts b/src/relay/relay-pty-source-restore-retry.test.ts new file mode 100644 index 00000000000..0eaf29d61c0 --- /dev/null +++ b/src/relay/relay-pty-source-restore-retry.test.ts @@ -0,0 +1,170 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + RelayDispatcher, + type RelayClientSessionIdentity, + type SinkWriteSettlement +} from './dispatcher' +import { encodeJsonRpcFrame, MessageType } from './protocol' +import { RelayPtySourcePublication } from './relay-pty-source-publication' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +const endpointIdentity: RelayClientSessionIdentity = { + principal: 'endpoint-principal', + authenticated: true, + allowSessionOwner: true, + authenticationKind: 'endpoint-credential' +} + +function requestFrame(id: number, method: string, params: Record): Buffer { + return encodeJsonRpcFrame({ jsonrpc: '2.0', id, method, params }, id, 0) +} + +function message(buffer: Buffer): Record | null { + if (buffer[0] !== MessageType.Regular) { + return null + } + const length = buffer.readUInt32BE(9) + return JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')) +} + +function responseResult( + writes: readonly Buffer[], + id: number +): Record | undefined { + const response = writes.map(message).find((entry) => entry?.id === id) + return response?.result as Record | undefined +} + +async function flushRequests(): Promise { + await new Promise((resolve) => setImmediate(resolve)) +} + +describe('relay PTY source restore retry', () => { + let dispatcher: RelayDispatcher | null = null + + afterEach(() => { + dispatcher?.dispose() + dispatcher = null + }) + + it('mints a fresh activation after invalid-checkpoint restore and retry', async () => { + const primaryWrites: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, settle) => { + primaryWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + let publication: RelayPtySourcePublication + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', undefined, (id) => + publication.onCreditAvailable(id) + ) + publication = new RelayPtySourcePublication(dispatcher, adapter, () => {}) + dispatcher.feed( + requestFrame(1, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const initialSettlements: ((result: SinkWriteSettlement) => void)[] = [] + expect( + publication.activate('pty-1', 'incarnation-1', { + clientId: 1, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => initialSettlements.push(callback) + }) + ).toBe('opened') + initialSettlements[0]({ ok: true }) + expect(publication.publish('pty-1', { data: 'old' }, false)).toBe(true) + const oldData = primaryWrites.map(message).find((entry) => entry?.method === 'pty.data')! + .params as Record + const oldGrant = responseResult(primaryWrites, 1)! + dispatcher.invalidateClient() + + const recoveredWrites: Buffer[] = [] + const recoveredClientId = dispatcher.attachClient( + (data, settle) => { + recoveredWrites.push(Buffer.from(data)) + settle({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + recoveredClientId, + requestFrame(2, 'pty.openClient', { + protocolVersion: 1, + clientInstanceId: 'client-1', + requestedRole: 'session-owner', + resume: { + ownerGeneration: oldGrant.ownerGeneration, + ownerLease: oldGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const restoreSettlements: ((result: SinkWriteSettlement) => void)[] = [] + expect( + publication.activate( + 'pty-1', + 'incarnation-1', + { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => restoreSettlements.push(callback) + }, + { + status: 'checkpoint', + deliveryToken: String(oldData.deliveryToken), + clientGeneration: Number(oldData.clientGeneration), + ownerGeneration: Number(oldData.ownerGeneration), + ptyIncarnation: 'incarnation-1', + acceptedSourceEndSu: 2 + } + ) + ).toMatchObject({ status: 'restoreRequired' }) + expect(publication.accepts('pty-1')).toBe(true) + for (const settle of restoreSettlements) { + settle({ ok: true }) + } + expect(publication.accepts('pty-1')).toBe(false) + + const retrySettlements: ((result: SinkWriteSettlement) => void)[] = [] + expect( + publication.activate('pty-1', 'incarnation-1', { + clientId: recoveredClientId, + isStale: () => false, + sessionIdentity: endpointIdentity, + onResponseSettled: (callback) => retrySettlements.push(callback) + }) + ).toBe('opened') + const sourceActivation = publication.receivingActivation('pty-1', recoveredClientId) + expect(sourceActivation).toMatchObject({ + status: 'pending', + checkpointSourceEndSu: 0, + recoveryEndSu: 0 + }) + expect(sourceActivation?.deliveryToken).not.toBe(oldData.deliveryToken) + retrySettlements[0]({ ok: true }) + + expect(publication.publish('pty-1', { data: 'live' }, false)).toBe(true) + const sourceFrames = recoveredWrites + .map(message) + .filter((entry) => entry?.method === 'pty.data') + expect(sourceFrames).toHaveLength(1) + expect(sourceFrames[0]?.params).toMatchObject({ + data: 'live', + deliveryToken: sourceActivation?.deliveryToken + }) + }) +}) diff --git a/src/relay/relay-pty-source-send-scheduler.ts b/src/relay/relay-pty-source-send-scheduler.ts new file mode 100644 index 00000000000..59402f5e5ba --- /dev/null +++ b/src/relay/relay-pty-source-send-scheduler.ts @@ -0,0 +1,290 @@ +import type { PtySourceDeliveryIdentity } from '../shared/pty-source-credit-contract' +import type { PtySourceRecoveryCheckpoint } from '../shared/pty-source-recovery-contract' +import type { PtySourceReceivingActivation } from '../shared/pty-source-receiving-activation' +import type { RelayDispatcher, SinkWriteSettlement } from './dispatcher' +import { + PTY_SOURCE_SCHEDULER_MAX_FRAMES, + PTY_SOURCE_SCHEDULER_MAX_SU +} from './pty-source-credit-scheduler' +import type { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' +import { completePtySourceRecovery } from './relay-pty-source-recovery-completion' + +export type RelayPtySourceDeliveryRecord = { + clientId: number + identity: PtySourceDeliveryIdentity + sourceActivation: PtySourceReceivingActivation + displayEnd: number + activating: boolean + activationRecoveryRequest: PtySourceRecoveryCheckpoint | null + sealed: boolean + legacyExitAccepted: boolean + sourceExitState: 'idle' | 'pending' | 'published' + sending: boolean + turnFrames: number + turnSourceSu: number + turnScheduled: boolean + sendWaiters: Set<() => void> + recoveryCheckpointSourceEndSu: number | null + recoveryEndSu: number | null + recoveryCompletionPending: boolean + restoreRequired: boolean + rotationPending: boolean +} + +export type RelayPtySourcePublicationCounters = { + opened: number + rotated: number + appendDenied: number + sendCommitted: number + sendRolledBack: number + exitCommitted: number + exitRolledBack: number +} + +const PTY_SOURCE_FRAME_MAX_SU = 16 * 1024 + +export function onceSinkSettlement( + callback: (result: SinkWriteSettlement) => void +): (result: SinkWriteSettlement) => void { + let settled = false + return (result) => { + if (settled) { + return + } + settled = true + callback(result) + } +} + +export class RelayPtySourceSendScheduler { + private readonly removeCompletionCapacityListener: () => void + + constructor( + private readonly dispatcher: RelayDispatcher, + private readonly session: SshPtyConsumerSessionAdapter, + private readonly deliveries: Map, + private readonly counters: RelayPtySourcePublicationCounters, + private readonly onCapacity: (id: string) => void + ) { + this.removeCompletionCapacityListener = dispatcher.onLegacyPtyCapacity(() => { + for (const record of deliveries.values()) { + this.completeRecoveryIfReady(record) + } + }) + } + + async waitForPendingSend(id: string, timeoutMs = 5_000): Promise { + const record = this.deliveries.get(id) + if (!record) { + return true + } + record.rotationPending = true + if (!record.sending) { + return true + } + return new Promise((resolve) => { + const settle = (settled: boolean): void => { + clearTimeout(timer) + record.sendWaiters.delete(onSettled) + resolve(settled) + } + const onSettled = (): void => settle(true) + const timer = setTimeout(() => settle(false), timeoutMs) + timer.unref?.() + record.sendWaiters.add(onSettled) + }) + } + + onCreditAvailable(id: string): void { + const record = this.deliveries.get(id) + if (!record || record.restoreRequired) { + return + } + if (this.pruneClosed(id, record)) { + this.onCapacity(id) + return + } + this.pump(record) + this.onCapacity(id) + } + + getDebugSnapshot() { + let active = 0 + let activating = 0 + let sealedUnsettled = 0 + let outstandingSourceUnits = 0 + for (const record of this.deliveries.values()) { + const snapshot = this.session.sourceDeliverySnapshot(record.identity) + outstandingSourceUnits += snapshot.sentEndSu - snapshot.creditedEndSu + if (record.activating) { + activating++ + } else if (record.sealed && snapshot.state !== 'closed') { + sealedUnsettled++ + } else if (snapshot.state === 'active') { + active++ + } + } + return Object.freeze({ + active, + activating, + sealedUnsettled, + outstandingSourceUnits, + ...this.counters + }) + } + + dispose(): void { + this.removeCompletionCapacityListener() + for (const record of this.deliveries.values()) { + this.session.cancelDelivery(record.identity, 'source-publication-disposed') + this.wakeSendWaiters(record) + } + this.deliveries.clear() + } + + pump(record: RelayPtySourceDeliveryRecord): void { + if ( + record.activating || + record.sending || + record.turnScheduled || + record.restoreRequired || + record.rotationPending || + this.deliveries.get(record.identity.id) !== record + ) { + return + } + if ( + record.recoveryEndSu !== null && + this.session.sourceDeliverySnapshot(record.identity).sentEndSu >= record.recoveryEndSu + ) { + this.completeRecoveryIfReady(record) + return + } + if ( + record.turnFrames >= PTY_SOURCE_SCHEDULER_MAX_FRAMES || + record.turnSourceSu >= PTY_SOURCE_SCHEDULER_MAX_SU + ) { + record.turnScheduled = true + setImmediate(() => { + record.turnScheduled = false + record.turnFrames = 0 + record.turnSourceSu = 0 + if (this.deliveries.get(record.identity.id) !== record || record.restoreRequired) { + return + } + this.pump(record) + this.onCapacity(record.identity.id) + }) + return + } + const snapshot = this.session.sourceDeliverySnapshot(record.identity) + const encodedDataBudget = this.dispatcher.producerDataBudget( + 'pty.data', + { + id: record.identity.id, + rawLength: PTY_SOURCE_FRAME_MAX_SU, + transformed: false, + deliveryToken: record.identity.deliveryToken, + clientGeneration: record.identity.clientGeneration, + ownerGeneration: record.identity.ownerGeneration, + ptyIncarnation: record.identity.ptyIncarnation, + sourceEndSu: snapshot.receivedEndSu, + sourceLengthSu: PTY_SOURCE_FRAME_MAX_SU + }, + record.clientId + ) + const maxSourceSu = Math.min( + PTY_SOURCE_FRAME_MAX_SU, + Math.max(1, Math.floor(Math.max(0, encodedDataBudget - 32) / 6)) + ) + const reservation = this.session.reserveSourceSend(record.identity, maxSourceSu) + if (!reservation) { + return + } + record.sending = true + const sourceLengthSu = reservation.span.sourceEndSu - reservation.span.sourceStartSu + const settle = onceSinkSettlement((result) => { + record.sending = false + this.wakeSendWaiters(record) + if (this.deliveries.get(record.identity.id) !== record || record.restoreRequired) { + this.onCapacity(record.identity.id) + return + } + if (result.ok) { + this.session.commitSourceSend(reservation) + record.turnFrames++ + record.turnSourceSu += sourceLengthSu + this.counters.sendCommitted++ + this.completeRecoveryIfReady(record) + } else { + this.session.rollbackSourceSend(reservation) + this.counters.sendRolledBack++ + } + this.pruneClosed(record.identity.id, record) + this.onCapacity(record.identity.id) + if (result.ok && this.deliveries.get(record.identity.id) === record) { + this.pump(record) + } + }) + const accepted = this.dispatcher.tryNotifyPtyDataToClient( + record.clientId, + { + id: reservation.identity.id, + data: reservation.span.data, + rawLength: sourceLengthSu, + transformed: reservation.span.transform.transformed, + deliveryToken: reservation.identity.deliveryToken, + clientGeneration: reservation.identity.clientGeneration, + ownerGeneration: reservation.identity.ownerGeneration, + ptyIncarnation: reservation.identity.ptyIncarnation, + sourceEndSu: reservation.span.sourceEndSu, + sourceLengthSu + }, + settle + ) + if (!accepted) { + settle({ ok: false, error: new Error('PTY source publication was not admitted') }) + } + } + + completeRecoveryIfReady(record: RelayPtySourceDeliveryRecord): void { + completePtySourceRecovery({ + record, + deliveries: this.deliveries, + dispatcher: this.dispatcher, + session: this.session, + onCompleted: (id) => { + this.onCapacity(id) + if (this.deliveries.get(id) === record) { + this.pump(record) + } + } + }) + } + + pruneClosed(id: string, record: RelayPtySourceDeliveryRecord): boolean { + if (this.session.sourceDeliverySnapshot(record.identity).state !== 'closed') { + return false + } + if (this.deliveries.get(id) === record) { + this.deliveries.delete(id) + } + return true + } + + wakeSendWaiters(record: RelayPtySourceDeliveryRecord): void { + for (const resolve of record.sendWaiters) { + resolve() + } + record.sendWaiters.clear() + } + + releaseRotationFence(record: RelayPtySourceDeliveryRecord | undefined): void { + if (!record?.rotationPending) { + return + } + record.rotationPending = false + this.pump(record) + this.onCapacity(record.identity.id) + } +} diff --git a/src/relay/relay-watcher-event-emitter.test.ts b/src/relay/relay-watcher-event-emitter.test.ts new file mode 100644 index 00000000000..ae5781c5745 --- /dev/null +++ b/src/relay/relay-watcher-event-emitter.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { RelayDispatcher } from './dispatcher' +import { HEADER_LENGTH, parseJsonRpcMessage } from './protocol' +import { emitRelayWatcherEvents, emitRelayWatcherOverflow } from './relay-watcher-event-emitter' + +function frameMethod(frame: Buffer): string { + const payloadLength = frame.readUInt32BE(9) + const message = parseJsonRpcMessage(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + payloadLength)) + return 'method' in message ? message.method : '' +} + +describe('relay watcher writer admission', () => { + it('keeps watcher batches on the bounded ordinary lane under saturation', () => { + const frames: Buffer[] = [] + const drainWaiters: (() => void)[] = [] + let saturate = true + const dispatcher = new RelayDispatcher( + (frame) => { + frames.push(Buffer.from(frame)) + if (saturate) { + saturate = false + return false + } + return true + }, + { + waitWriteDrain: (callback) => { + drainWaiters.push(callback) + return () => { + const index = drainWaiters.indexOf(callback) + if (index >= 0) { + drainWaiters.splice(index, 1) + } + } + } + } + ) + + try { + emitRelayWatcherEvents(dispatcher, false, [ + { type: 'create', path: '/workspace/first', isDirectory: false } + ]) + emitRelayWatcherOverflow(dispatcher, '/workspace', false) + dispatcher.notifyClient(1, 'control.event') + + expect(frames.map(frameMethod)).toEqual(['fs.changed']) + drainWaiters.shift()?.() + expect(frames.map(frameMethod)).toEqual(['fs.changed', 'control.event', 'fs.changed']) + } finally { + dispatcher.dispose() + } + }) +}) diff --git a/src/relay/relay-watcher-event-emitter.ts b/src/relay/relay-watcher-event-emitter.ts index 46515e67959..1a575c360fa 100644 --- a/src/relay/relay-watcher-event-emitter.ts +++ b/src/relay/relay-watcher-event-emitter.ts @@ -9,6 +9,7 @@ export function emitRelayWatcherEvents( if (closed || events.length === 0) { return } + // Watcher floods stay on the bounded producer lane; its cap closes the stale generation. dispatcher.notify('fs.changed', { events: events.map((event) => ({ kind: event.type, diff --git a/src/relay/relay.ts b/src/relay/relay.ts index 6daf87b309c..ab485f71312 100644 --- a/src/relay/relay.ts +++ b/src/relay/relay.ts @@ -10,7 +10,7 @@ import { createServer, createConnection, type Socket, type Server } from 'node:net' import { join } from 'node:path' -import { unlinkSync, existsSync, statSync } from 'node:fs' +import { unlinkSync, existsSync, statSync, readFileSync, chmodSync } from 'node:fs' import { RELAY_SENTINEL, FrameDecoder, @@ -53,6 +53,9 @@ import { remoteCliRequestTimeoutMs } from './remote-cli-timeout' import { shouldReadRemoteCliStdin } from './remote-cli-stdin' import { registerManagedHookInstaller } from './managed-hook-installer' import { registerRelayPluginHostCallHandlers } from './plugin-host-call-handler' +import { DispatcherClientWriter } from './dispatcher-client-writer' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' +import { RelayPtySourcePublication } from './relay-pty-source-publication' const DEFAULT_GRACE_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000 const SOCK_NAME = 'relay.sock' @@ -106,6 +109,7 @@ function parseArgs(argv: string[]): { sockPath: string endpointDir?: string logFile?: string + credentialFile?: string } { let graceTimeMs = DEFAULT_GRACE_MS let connectMode = false @@ -114,6 +118,7 @@ function parseArgs(argv: string[]): { let sockPath = '' let endpointDir: string | undefined let logFile: string | undefined + let credentialFile: string | undefined for (let i = 2; i < argv.length; i++) { if (argv[i] === '--grace-time' && argv[i + 1]) { const parsed = Number.parseInt(argv[i + 1], 10) @@ -137,20 +142,65 @@ function parseArgs(argv: string[]): { } else if (argv[i] === '--log-file' && argv[i + 1]) { logFile = argv[i + 1] i++ + } else if (argv[i] === '--credential-file' && argv[i + 1]) { + credentialFile = argv[i + 1] + i++ } } if (!sockPath) { sockPath = join(process.cwd(), SOCK_NAME) } - return { graceTimeMs, connectMode, detached, cliMode, sockPath, endpointDir, logFile } + return { + graceTimeMs, + connectMode, + detached, + cliMode, + sockPath, + endpointDir, + logFile, + credentialFile + } +} + +function readEndpointCredential(credentialFile: string | undefined): string | undefined { + if (!credentialFile) { + return undefined + } + const credential = readFileSync(credentialFile, 'utf8').trim() + if (!/^[A-Za-z0-9_-]{32,256}$/.test(credential)) { + throw new Error('Relay endpoint credential is missing or invalid') + } + if (process.platform !== 'win32') { + chmodSync(credentialFile, 0o600) + } + return credential } // ── Connect mode ───────────────────────────────────────────────────── // Why: --connect bridges a new SSH channel's stdin/stdout to the existing relay's socket so the client keeps talking to the process that owns the live PTYs. -function runConnectMode(sockPath: string): void { +function runConnectMode(sockPath: string, endpointCredential?: string): void { const myVersion = readLaunchVersion() const sock = createConnection({ path: sockPath }) + const stdoutWriter = new DispatcherClientWriter( + (data, onSettled) => + process.stdout.write(data, (error) => { + onSettled(error ? { ok: false, error } : { ok: true }) + }), + { + supportsWriteCallback: true, + writableLength: () => process.stdout.writableLength, + writableHighWaterMark: () => process.stdout.writableHighWaterMark, + waitWriteDrain: (callback) => { + process.stdout.once('drain', callback) + return () => process.stdout.off('drain', callback) + } + }, + () => { + sock.destroy() + process.exit(1) + } + ) const connectTimeout = setTimeout(() => { process.stderr.write(`[relay-connect] Connection timed out after ${CONNECT_TIMEOUT_MS}ms\n`) @@ -160,24 +210,58 @@ function runConnectMode(sockPath: string): void { sock.on('connect', () => { clearTimeout(connectTimeout) - runConnectHandshake(sock, myVersion, { - onAccepted: (leftover: Buffer) => { - // Why: write RELAY_SENTINEL only after the handshake passes, so a version mismatch is a clean exit-42 instead of a false sentinel + channel drop. - process.stdout.write(RELAY_SENTINEL) - // Why: forward handshake-buffered leftover bytes before sock.pipe(process.stdout) so the downstream mux sees them in order. - if (leftover.length > 0) { - process.stdout.write(leftover) + runConnectHandshake( + sock, + myVersion, + { + onAccepted: (leftover: Buffer) => { + stdoutWriter.enqueue('control', () => Buffer.from(RELAY_SENTINEL), RELAY_SENTINEL.length) + if (leftover.length > 0) { + stdoutWriter.enqueue('control', () => leftover, leftover.length) + } + process.stdin.pipe(sock) + sock.on('data', (data: Buffer) => { + sock.pause() + let offset = 0 + const writeNext = (): void => { + if (offset >= data.length) { + sock.resume() + return + } + const bytes = Math.min(stdoutWriter.producerFrameCapacity, data.length - offset) + if (bytes <= 0) { + stdoutWriter.close(new Error('Relay stdout has no producer capacity')) + return + } + const chunk = data.subarray(offset, offset + bytes) + if ( + !stdoutWriter.enqueue( + 'ordinary', + () => chunk, + chunk.length, + (result) => { + if (!result.ok) { + return + } + offset += bytes + writeNext() + } + ) + ) { + stdoutWriter.close(new Error('Relay stdout bridge capacity exceeded')) + } + } + writeNext() + }) } - process.stdin.pipe(sock) - sock.pipe(process.stdout) - } - }) + }, + endpointCredential + ) }) // Why: Node swallows EPIPE on stdout, so the bridge would zombie and drop frames; exit on stdout error so the relay enters grace promptly. process.stdout.on('error', () => { - sock.destroy() - process.exit(1) + stdoutWriter.close(new Error('Relay stdout closed')) }) sock.on('error', (err) => { @@ -186,15 +270,36 @@ function runConnectMode(sockPath: string): void { process.exit(1) }) - sock.on('close', () => { + sock.on('close', async () => { + await stdoutWriter.waitForIdle() process.exit(0) }) } -async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { +async function runOrcaCliMode( + sockPath: string, + argv: string[], + endpointCredential?: string +): Promise { const myVersion = readLaunchVersion() const stdin = shouldReadRemoteCliStdin(argv) ? await readOrcaCliStdin() : undefined const sock = createConnection({ path: sockPath }) + const stdoutWriter = new DispatcherClientWriter( + (data, onSettled) => + process.stdout.write(data, (error) => { + onSettled(error ? { ok: false, error } : { ok: true }) + }), + { + supportsWriteCallback: true, + writableLength: () => process.stdout.writableLength, + writableHighWaterMark: () => process.stdout.writableHighWaterMark, + waitWriteDrain: (callback) => { + process.stdout.once('drain', callback) + return () => process.stdout.off('drain', callback) + } + }, + () => process.exit(1) + ) let nextSeq = 1 let highestReceivedSeq = 0 const requestId = 1 @@ -245,32 +350,40 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { result: { stdout?: unknown; stderr?: unknown }, onFlushed: (error?: Error) => void ): void => { - const writes = [ - [process.stdout, result.stdout], - [process.stderr, result.stderr] - ].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string') - if (writes.length === 0) { - onFlushed() - return - } - let pending = writes.length + let pending = 0 let completed = false - for (const [stream, output] of writes) { - stream.write(output, (error) => { - if (completed) { - return - } - if (error) { - completed = true - onFlushed(error) - return - } - pending -= 1 - if (pending === 0) { - completed = true - onFlushed() - } - }) + const settle = (error?: Error): void => { + if (completed) { + return + } + if (error) { + completed = true + onFlushed(error) + return + } + pending -= 1 + if (pending === 0) { + completed = true + onFlushed() + } + } + if (typeof result.stdout === 'string' && result.stdout.length > 0) { + pending += 1 + const output = Buffer.from(result.stdout) + stdoutWriter.enqueue( + 'control', + () => output, + output.length, + (settlement) => settle(settlement.ok ? undefined : settlement.error) + ) + } + if (typeof result.stderr === 'string' && result.stderr.length > 0) { + pending += 1 + process.stderr.write(result.stderr, 'utf8', (error) => settle(error ?? undefined)) + } + if (pending === 0) { + completed = true + onFlushed() } } @@ -327,17 +440,22 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { sock.on('connect', () => { clearTimeout(connectTimeout) - runConnectHandshake(sock, myVersion, { - onAccepted: (leftover) => { - if (leftover.length > 0) { - decoder.feed(leftover) + runConnectHandshake( + sock, + myVersion, + { + onAccepted: (leftover) => { + if (leftover.length > 0) { + decoder.feed(leftover) + } + sock.on('data', (chunk) => + decoder.feed(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + ) + sendRequest() } - sock.on('data', (chunk) => - decoder.feed(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) - ) - sendRequest() - } - }) + }, + endpointCredential + ) }) sock.on('error', (err) => { @@ -361,17 +479,29 @@ async function readOrcaCliStdin(): Promise { // ── Normal mode ────────────────────────────────────────────────────── async function main(): Promise { - const { graceTimeMs, connectMode, detached, cliMode, sockPath, endpointDir, logFile } = parseArgs( - process.argv - ) + const { + graceTimeMs, + connectMode, + detached, + cliMode, + sockPath, + endpointDir, + logFile, + credentialFile + } = parseArgs(process.argv) + const endpointCredential = readEndpointCredential(credentialFile) if (connectMode) { - runConnectMode(sockPath) + runConnectMode(sockPath, endpointCredential) return } if (cliMode) { const marker = process.argv.indexOf('--orca-cli') - await runOrcaCliMode(sockPath, marker >= 0 ? process.argv.slice(marker + 1) : []) + await runOrcaCliMode( + sockPath, + marker >= 0 ? process.argv.slice(marker + 1) : [], + endpointCredential + ) return } @@ -425,29 +555,42 @@ async function main(): Promise { } process.stdout.on('drain', flushStdoutDrainWaiters) const dispatcher = new RelayDispatcher( - (data) => { + (data, onSettled) => { if (!stdoutAlive) { - return + onSettled({ ok: false, error: new Error('Relay stdout is closed') }) + return false } try { - // Why: surface Node's backpressure so bulk frames (fs.streamChunk) wait for drain instead of queueing ahead of interactive pty.data. - return process.stdout.write(data) - } catch { + return process.stdout.write(data, (error) => { + onSettled(error ? { ok: false, error } : { ok: true }) + }) + } catch (error) { stdoutAlive = false flushStdoutDrainWaiters() - return undefined + onSettled({ + ok: false, + error: error instanceof Error ? error : new Error(String(error)) + }) + return false } }, { + supportsWriteCallback: true, + writableLength: () => process.stdout.writableLength, + writableHighWaterMark: () => process.stdout.writableHighWaterMark, waitWriteDrain: (cb) => { if (!stdoutAlive) { cb() return } stdoutDrainWaiters.add(cb) + return () => stdoutDrainWaiters.delete(cb) } - } + }, + undefined, + { pauseReads: () => process.stdin.pause(), resumeReads: () => process.stdin.resume() } ) + const launchVersion = readLaunchVersion() const context = new RelayContext() @@ -477,6 +620,18 @@ async function main(): Promise { }) const ptyHandler = new PtyHandler(dispatcher, graceTimeMs) + const ptyConsumerSessionAdapter = new SshPtyConsumerSessionAdapter( + dispatcher, + launchVersion, + (id, paused) => ptyHandler.setConsumerDeliveryPaused(id, paused), + (id) => ptyHandler.handleSourceCreditAvailable(id) + ) + const ptySourcePublication = new RelayPtySourcePublication( + dispatcher, + ptyConsumerSessionAdapter, + (id) => ptyHandler.handleSourcePublicationCapacity(id) + ) + ptyHandler.setSourcePublication(ptySourcePublication) const fsHandler = new FsHandler(dispatcher, context) const watchRegistry = fsHandler.getWatchRegistry() ptyHandler.setWorktreeRemovalCoordinator(watchRegistry) @@ -651,7 +806,6 @@ async function main(): Promise { const socketClients = new Map() let socketServer: Server | null = null - const launchVersion = readLaunchVersion() const startedAt = Date.now() let acceptedSocketConnections = 0 let hasAcceptedSocketClient = false @@ -667,6 +821,11 @@ async function main(): Promise { ptys: { active: ptyHandler.activePtyCount }, + ptySourceCredit: { + enabled: true, + session: ptyConsumerSessionAdapter.getDebugSnapshot(), + publication: ptySourcePublication.getDebugSnapshot() + }, socket: { path: sockPath, owned: ownsSocketPath, @@ -714,20 +873,38 @@ async function main(): Promise { sock.on('close', flushSockDrainWaiters) sock.on('error', flushSockDrainWaiters) const clientId = dispatcher.attachClient( - (data) => { + (data, onSettled) => { if (!sock.destroyed) { - return sock.write(data) + return sock.write(data, (error) => { + onSettled(error ? { ok: false, error } : { ok: true }) + }) } - return undefined + onSettled({ ok: false, error: new Error('Relay socket is closed') }) + return false }, { + supportsWriteCallback: true, + writableLength: () => sock.writableLength, + writableHighWaterMark: () => sock.writableHighWaterMark, + close: () => sock.destroy(), waitWriteDrain: (cb) => { if (sock.destroyed) { cb() return } sockDrainWaiters.add(cb) + return () => sockDrainWaiters.delete(cb) } + }, + { + principal: `relay-endpoint:${launchVersion}`, + authenticated: endpointCredential !== undefined, + allowSessionOwner: endpointCredential !== undefined, + authenticationKind: endpointCredential ? 'endpoint-credential' : 'unproved' + }, + { + pauseReads: () => sock.pause(), + resumeReads: () => sock.resume() } ) socketClients.set(sock, clientId) @@ -746,7 +923,11 @@ async function main(): Promise { async function startSocketServer(): Promise { const server = createServer((sock) => { // Why: pre-dispatcher version handshake — see relay-handshake.ts. - setupDaemonHandshake(sock, { launchVersion, onAccepted: attachAcceptedSocket }) + setupDaemonHandshake(sock, { + launchVersion, + endpointCredential, + onAccepted: attachAcceptedSocket + }) // Why: destroy on 'end' (FIN from --connect's dying channel) so the 'close' handler fires promptly and the daemon enters grace. sock.on('end', () => { @@ -939,7 +1120,6 @@ async function main(): Promise { if (detached) { // Why: detached stdin is /dev/null, so listening would EOF → grace → shutdown before --connect arrives; use the socket instead. - stdoutAlive = false startGrace('detached startup') } else { process.stdin.on('data', (chunk: Buffer) => { @@ -1011,8 +1191,11 @@ async function main(): Promise { relayLogLine(`[relay] Process exiting with code ${code}`) }) - // Why: the client waits for this exact sentinel string before sending framed data. - process.stdout.write(RELAY_SENTINEL) + dispatcher.writePrimaryBytes(Buffer.from(RELAY_SENTINEL)) + if (detached) { + stdoutAlive = false + dispatcher.invalidateClient() + } } function cleanupSocket(sockPath: string): void { diff --git a/src/relay/ssh-pty-consumer-session-adapter.test.ts b/src/relay/ssh-pty-consumer-session-adapter.test.ts new file mode 100644 index 00000000000..a0f040814ed --- /dev/null +++ b/src/relay/ssh-pty-consumer-session-adapter.test.ts @@ -0,0 +1,601 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { PTY_CONSUMER_OWNER_GRACE_MS } from '../shared/pty-consumer-session' +import { RelayDispatcher, type RelayClientSessionIdentity } from './dispatcher' +import { encodeJsonRpcFrame, MessageType } from './protocol' +import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter' + +const endpointIdentity: RelayClientSessionIdentity = { + principal: 'endpoint-principal', + authenticated: true, + allowSessionOwner: true, + authenticationKind: 'endpoint-credential' +} + +function openFrame(id: number, overrides: Record = {}): Buffer { + return encodeJsonRpcFrame( + { + jsonrpc: '2.0', + id, + method: 'pty.openClient', + params: { + protocolVersion: 1, + clientInstanceId: `client-${id}`, + requestedRole: 'session-owner', + ...overrides + } + }, + 1, + 0 + ) +} + +function responseResult(buffer: Buffer): Record { + expect(buffer[0]).toBe(MessageType.Regular) + const length = buffer.readUInt32BE(9) + return JSON.parse(buffer.subarray(13, 13 + length).toString('utf8')).result +} + +async function flushRequests(): Promise { + await new Promise((resolve) => setImmediate(resolve)) +} + +describe('SshPtyConsumerSessionAdapter', () => { + let dispatcher: RelayDispatcher | null = null + + afterEach(() => { + dispatcher?.dispose() + dispatcher = null + vi.useRealTimers() + }) + + it('does not activate owner authority until the grant write settles', async () => { + const firstWrites: Buffer[] = [] + const firstSettlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + firstWrites.push(Buffer.from(data)) + firstSettlements.push(onSettled) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + + dispatcher.feed(openFrame(1)) + await flushRequests() + + const secondWrites: Buffer[] = [] + const secondId = dispatcher.attachClient( + (data, onSettled) => { + secondWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + { ...endpointIdentity, principal: 'competitor' } + ) + dispatcher.feedClient(secondId, openFrame(2)) + await flushRequests() + + expect(responseResult(firstWrites[0])).toMatchObject({ + role: 'session-owner', + ownerGeneration: 1 + }) + expect(responseResult(secondWrites[0])).toMatchObject({ role: 'subscriber' }) + firstSettlements[0]({ ok: true }) + }) + + it('rolls back owner election when the grant write fails', async () => { + dispatcher = new RelayDispatcher( + (_data, onSettled) => { + onSettled({ ok: false, error: new Error('send failed') }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed(openFrame(1)) + await flushRequests() + + const retryWrites: Buffer[] = [] + const retryId = dispatcher.attachClient( + (data, onSettled) => { + retryWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient(retryId, openFrame(2, { clientInstanceId: 'client-1' })) + await flushRequests() + + expect(responseResult(retryWrites[0])).toMatchObject({ + role: 'session-owner', + ownerGeneration: 2 + }) + }) + + it('rejects an unproved constructor stream as an owner principal', async () => { + const writes: Buffer[] = [] + dispatcher = new RelayDispatcher((data, onSettled) => { + writes.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + + dispatcher.feed(openFrame(1)) + await flushRequests() + + const response = JSON.parse( + writes[0].subarray(13, 13 + writes[0].readUInt32BE(9)).toString('utf8') + ) + expect(response.error.message).toContain('authentication') + }) + + it('rejects an invalid requested role instead of promoting it to owner', async () => { + const writes: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + writes.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + + dispatcher.feed(openFrame(1, { requestedRole: 'administrator' })) + await flushRequests() + + const response = JSON.parse( + writes[0].subarray(13, 13 + writes[0].readUInt32BE(9)).toString('utf8') + ) + expect(response.error.message).toContain('requestedRole') + }) + + it('generation-fences per-PTY delivery pause notifications', async () => { + const setPaused: { id: string; paused: boolean }[] = [] + dispatcher = new RelayDispatcher( + (_data, onSettled) => { + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', (id, paused) => { + setPaused.push({ id, paused }) + }) + dispatcher.feed(openFrame(1)) + await flushRequests() + + dispatcher.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.setDeliveryPaused', + params: { + id: 'pty-1', + paused: true, + clientGeneration: 1, + ownerGeneration: 1 + } + }, + 2, + 0 + ) + ) + dispatcher.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.setDeliveryPaused', + params: { + id: 'pty-1', + paused: false, + clientGeneration: 99, + ownerGeneration: 1 + } + }, + 3, + 0 + ) + ) + + expect(setPaused).toEqual([{ id: 'pty-1', paused: true }]) + }) + + it('token-fences V1 pause and clears an owned pause before detach retention', async () => { + const setPaused: { id: string; paused: boolean }[] = [] + dispatcher = new RelayDispatcher( + (_data, onSettled) => { + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', (id, paused) => { + setPaused.push({ id, paused }) + }) + dispatcher.feed( + openFrame(1, { + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + + for (const [sequence, deliveryToken] of [ + [2, 'stale-token'], + [3, identity.deliveryToken] + ] as const) { + dispatcher.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.setDeliveryPaused', + params: { + id: identity.id, + paused: true, + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration, + deliveryToken + } + }, + sequence, + 0 + ) + ) + } + dispatcher.invalidateClient() + + expect(setPaused).toEqual([ + { id: 'pty-1', paused: true }, + { id: 'pty-1', paused: false } + ]) + }) + + it('clears the exact token pause before cancellation cleanup', async () => { + const setPaused: { id: string; paused: boolean }[] = [] + dispatcher = new RelayDispatcher( + (_data, onSettled) => { + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', (id, paused) => { + setPaused.push({ id, paused }) + }) + dispatcher.feed( + openFrame(1, { + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + dispatcher.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.setDeliveryPaused', + params: { + id: identity.id, + paused: true, + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration, + deliveryToken: identity.deliveryToken + } + }, + 2, + 0 + ) + ) + + adapter.cancelDelivery(identity, 'restore-required') + + expect(setPaused).toEqual([ + { id: 'pty-1', paused: true }, + { id: 'pty-1', paused: false } + ]) + }) + + it('intersects the negotiated V1 source window without changing legacy omission', async () => { + const writes: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + writes.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed( + openFrame(1, { + capabilities: { + outputFlowControl: { versions: [1], requestedWindowSu: 512 * 1024 } + } + }) + ) + await flushRequests() + + expect(responseResult(writes[0])).toMatchObject({ + capabilities: { + outputFlowControl: { version: 1, windowSu: 256 * 1024 } + } + }) + }) + + it('keeps an old client token-free when the new relay supports V1', async () => { + const writes: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + writes.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed(openFrame(1)) + await flushRequests() + + const grant = responseResult(writes[0]) + expect(grant).not.toHaveProperty('capabilities') + expect(grant).not.toHaveProperty('deliveryToken') + }) + + it('accepts cumulative source ACKs only from the negotiated token owner', async () => { + dispatcher = new RelayDispatcher( + (_data, onSettled) => { + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed( + openFrame(1, { + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } } + }) + ) + await flushRequests() + const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + adapter.appendSource(identity, { + spanId: 'span-1', + data: 'data', + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + }) + const reservation = adapter.reserveSourceSend(identity)! + adapter.commitSourceSend(reservation) + + dispatcher.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + method: 'pty.ackData', + params: { + acknowledgements: [ + { + id: identity.id, + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration, + deliveryToken: identity.deliveryToken, + creditedEndSu: 4 + } + ] + } + }, + 2, + 0 + ) + ) + + expect(adapter.sourceDeliverySnapshot(identity).creditedEndSu).toBe(4) + }) + + it('returns exact token-scoped cancellation proof before local cleanup', async () => { + const writes: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + writes.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed( + openFrame(1, { + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + adapter.appendSource(identity, { + spanId: 'span-1', + data: 'data', + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + }) + adapter.commitSourceSend(adapter.reserveSourceSend(identity)!) + dispatcher.feed( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + id: 2, + method: 'pty.cancelDelivery', + params: { + id: identity.id, + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration, + deliveryToken: identity.deliveryToken + } + }, + 2, + 0 + ) + ) + await flushRequests() + + expect(responseResult(writes.at(-1)!)).toEqual({ + canceled: true, + sentEndSu: 4, + creditedEndSu: 0 + }) + expect(adapter.sourceDeliverySnapshot(identity).state).toBe('closed') + }) + + it('rotates reconnect ownership and transfers only exact retained recovery', async () => { + const firstWrites: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + firstWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed( + openFrame(1, { + clientInstanceId: 'stable-client', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + const firstGrant = responseResult(firstWrites[0]) + const oldIdentity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + adapter.appendSource(oldIdentity, { + spanId: 'span-1', + data: 'data', + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + }) + dispatcher.invalidateClient() + + const recoveredWrites: Buffer[] = [] + const recoveredClientId = dispatcher.attachClient( + (data, onSettled) => { + recoveredWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + recoveredClientId, + openFrame(2, { + clientInstanceId: 'stable-client', + resume: { + ownerGeneration: firstGrant.ownerGeneration, + ownerLease: firstGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + + const rotation = adapter.rotateDelivery(oldIdentity, recoveredClientId, 0) + expect(rotation.recovery.map((sourceSpan) => sourceSpan.data)).toEqual(['data']) + expect(rotation.identity).toMatchObject({ + providerGeneration: oldIdentity.providerGeneration, + clientGeneration: 2, + ownerGeneration: 2 + }) + expect(rotation.cancellation.replacementDeliveryToken).toBe(rotation.identity.deliveryToken) + }) + + it('keeps the old-token deadline after a resume grant until exact rotation', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }) + const firstWrites: Buffer[] = [] + dispatcher = new RelayDispatcher( + (data, onSettled) => { + firstWrites.push(Buffer.from(data)) + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed( + openFrame(1, { + clientInstanceId: 'stable-client', + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + const firstGrant = responseResult(firstWrites[0]) + const oldIdentity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + dispatcher.invalidateClient() + + const recoveredClientId = dispatcher.attachClient( + (_data, onSettled) => { + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + dispatcher.feedClient( + recoveredClientId, + openFrame(2, { + clientInstanceId: 'stable-client', + resume: { + ownerGeneration: firstGrant.ownerGeneration, + ownerLease: firstGrant.ownerLease + }, + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + vi.advanceTimersByTime(PTY_CONSUMER_OWNER_GRACE_MS) + + expect(adapter.sourceDeliverySnapshot(oldIdentity).state).toBe('closed') + }) + + it('closes retained source tokens when the dispatcher is disposed', async () => { + dispatcher = new RelayDispatcher( + (_data, onSettled) => { + onSettled({ ok: true }) + return true + }, + { supportsWriteCallback: true }, + endpointIdentity + ) + const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a') + dispatcher.feed( + openFrame(1, { + capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } } + }) + ) + await flushRequests() + const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')! + + dispatcher.dispose() + + expect(adapter.sourceDeliverySnapshot(identity).state).toBe('closed') + }) +}) diff --git a/src/relay/ssh-pty-consumer-session-adapter.ts b/src/relay/ssh-pty-consumer-session-adapter.ts new file mode 100644 index 00000000000..c5bedb37053 --- /dev/null +++ b/src/relay/ssh-pty-consumer-session-adapter.ts @@ -0,0 +1,293 @@ +import { + PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + PtyConsumerSession, + type PtyConsumerSessionGrant, + type PtyConsumerSessionHello +} from '../shared/pty-consumer-session' +import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../shared/pty-source-credit-contract' +import type { + PtySourceDeliveryIdentity, + PtySourceDeliverySnapshot, + PtySourceSpan, + PtySourceTransform +} from '../shared/pty-source-credit-contract' +import type { RelayClientSessionIdentity, RelayDispatcher, RequestContext } from './dispatcher' +import type { PtySourceSendReservation } from './pty-source-credit-ledger' +import { SshPtySourceCreditAdapter } from './ssh-pty-source-credit-adapter' + +export const SSH_PTY_OPEN_CLIENT_METHOD = 'pty.openClient' + +type OpenClientParams = PtyConsumerSessionHello & { + protocolVersion: number +} + +function parseOpenClientParams(params: Record): OpenClientParams { + const resume = + typeof params.resume === 'object' && params.resume !== null + ? (params.resume as Record) + : undefined + const capabilities = + typeof params.capabilities === 'object' && params.capabilities !== null + ? (params.capabilities as Record) + : undefined + const outputFlowControl = + typeof capabilities?.outputFlowControl === 'object' && capabilities.outputFlowControl !== null + ? (capabilities.outputFlowControl as Record) + : undefined + return { + protocolVersion: Number(params.protocolVersion), + clientInstanceId: String(params.clientInstanceId ?? ''), + requestedRole: String(params.requestedRole ?? '') as PtyConsumerSessionHello['requestedRole'], + ...(resume + ? { + resume: { + ownerGeneration: Number(resume.ownerGeneration), + ownerLease: String(resume.ownerLease ?? '') + } + } + : {}), + ...(outputFlowControl + ? { + capabilities: { + outputFlowControl: { + versions: Array.isArray(outputFlowControl.versions) + ? outputFlowControl.versions.map(Number) + : [], + requestedWindowSu: Number(outputFlowControl.requestedWindowSu) + } + } + } + : {}) + } +} + +function requireIdentity(context: RequestContext): RelayClientSessionIdentity { + if (!context.sessionIdentity) { + throw new Error('SSH PTY consumer transport identity is unavailable') + } + return context.sessionIdentity +} + +export class SshPtyConsumerSessionAdapter { + private readonly session: PtyConsumerSession + private readonly sourceCredit: SshPtySourceCreditAdapter + private readonly pausedDeliveryByPty = new Map() + + constructor( + dispatcher: RelayDispatcher, + serverBuildId: string, + private readonly setDeliveryPaused?: (id: string, paused: boolean) => void, + onSourceCreditAvailable?: (id: string) => void + ) { + this.sourceCredit = new SshPtySourceCreditAdapter( + (proof) => + dispatcher.notifyControl( + 'pty.deliveryCanceled', + proof as unknown as Record + ), + onSourceCreditAvailable + ) + this.session = new PtyConsumerSession({ + serverBuildId, + outputFlowControl: { versions: [1], maxWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU } + }) + dispatcher.onRequest(SSH_PTY_OPEN_CLIENT_METHOD, (params, context) => + this.openClient(params, context) + ) + dispatcher.onClientDetached((clientId) => { + const grant = this.session.activeGrant(String(clientId)) + if (grant) { + this.clearPausedForGrant(grant) + } + this.session.close(String(clientId)) + if (grant) { + this.sourceCredit.retainOrCloseOnDetach(grant) + } + }) + dispatcher.onNotification('pty.setDeliveryPaused', (params, context) => { + const grant = this.session.activeGrant(String(context.clientId)) + if ( + !grant || + grant.clientGeneration !== params.clientGeneration || + grant.ownerGeneration !== params.ownerGeneration || + typeof params.id !== 'string' || + typeof params.paused !== 'boolean' + ) { + return + } + if (grant.capabilities?.outputFlowControl) { + const token = typeof params.deliveryToken === 'string' ? params.deliveryToken : '' + const identity = this.sourceCredit.ownsDelivery(token, grant, params.id) + if (!identity) { + return + } + if (params.paused) { + this.pausedDeliveryByPty.set(params.id, identity) + } else if (this.pausedDeliveryByPty.get(params.id) !== identity) { + return + } else { + this.pausedDeliveryByPty.delete(params.id) + } + } + this.setDeliveryPaused?.(params.id, params.paused) + }) + dispatcher.onNotification('pty.ackData', (params, context) => { + this.sourceCredit.acknowledge(params, this.session.activeGrant(String(context.clientId))) + }) + dispatcher.onRequest('pty.cancelDelivery', async (params, context) => + this.sourceCredit.cancel(params, this.session.activeGrant(String(context.clientId))) + ) + dispatcher.onDisposed(() => { + for (const id of this.pausedDeliveryByPty.keys()) { + this.setDeliveryPaused?.(id, false) + } + this.pausedDeliveryByPty.clear() + this.sourceCredit.dispose() + }) + } + + openDelivery( + clientId: number, + id: string, + ptyIncarnation: string, + checkpointSourceEndSu = 0 + ): PtySourceDeliveryIdentity | null { + return this.sourceCredit.open( + this.session.activeGrant(String(clientId)), + id, + ptyIncarnation, + checkpointSourceEndSu + ) + } + + rotateDelivery( + oldIdentity: PtySourceDeliveryIdentity, + newClientId: number, + acceptedSourceEndSu: number + ) { + this.clearPausedIdentity(oldIdentity) + return this.sourceCredit.rotate( + oldIdentity, + this.session.activeGrant(String(newClientId)), + acceptedSourceEndSu + ) + } + + appendSource( + identity: PtySourceDeliveryIdentity, + input: Readonly<{ + spanId: string + data: string + displayStart: number + displayEnd: number + splittable: boolean + transform: PtySourceTransform + }> + ): PtySourceSpan { + return this.sourceCredit.append(identity, input) + } + + reserveSourceSend( + identity: PtySourceDeliveryIdentity, + maxSourceSu?: number + ): PtySourceSendReservation | null { + return this.sourceCredit.reserveSend(identity, maxSourceSu) + } + + commitSourceSend(reservation: PtySourceSendReservation): void { + this.sourceCredit.commitSend(reservation) + } + + rollbackSourceSend(reservation: PtySourceSendReservation): void { + this.sourceCredit.rollbackSend(reservation) + } + + sealDelivery(identity: PtySourceDeliveryIdentity): void { + this.sourceCredit.seal(identity) + } + + settleExitPublication( + identity: PtySourceDeliveryIdentity, + result: { ok: true } | { ok: false; error: Error } + ): void { + this.sourceCredit.settleExit(identity, result) + } + + sourceDeliverySnapshot(identity: PtySourceDeliveryIdentity): PtySourceDeliverySnapshot { + return this.sourceCredit.snapshot(identity) + } + + cancelDelivery(identity: PtySourceDeliveryIdentity, reason: string): void { + this.clearPausedIdentity(identity) + this.sourceCredit.cancelIdentity(identity, reason) + } + + getDebugSnapshot(): Readonly<{ + deliveryTokens: number + graceTimers: number + sourceSu: number + dataBytes: number + spans: number + }> { + return this.sourceCredit.retentionSnapshot() + } + + deliveryMode(clientId: number): 'source-owner' | 'legacy-owner' | 'subscriber' { + const grant = this.session.activeGrant(String(clientId)) + if (grant?.role !== 'session-owner') { + return 'subscriber' + } + return grant.capabilities?.outputFlowControl ? 'source-owner' : 'legacy-owner' + } + + private async openClient( + rawParams: Record, + context: RequestContext + ): Promise { + const params = parseOpenClientParams(rawParams) + if (params.protocolVersion !== PTY_CONSUMER_SESSION_PROTOCOL_VERSION) { + throw new Error( + `Unsupported pty.openClient protocol version: ${params.protocolVersion || 'missing'}` + ) + } + const identity = requireIdentity(context) + const admission = this.session.admit(params, { + connectionId: String(context.clientId), + principal: identity.principal, + authenticated: identity.authenticated, + allowSessionOwner: identity.allowSessionOwner + }) + if (!context.onResponseSettled) { + admission.rollbackPublication() + throw new Error('SSH PTY consumer response publication fence is unavailable') + } + context.onResponseSettled((result) => { + if (result.ok) { + admission.commitPublication() + } else { + admission.rollbackPublication() + } + }) + return admission.grant + } + + private clearPausedIdentity(identity: PtySourceDeliveryIdentity): void { + if (this.pausedDeliveryByPty.get(identity.id) === identity) { + this.pausedDeliveryByPty.delete(identity.id) + this.setDeliveryPaused?.(identity.id, false) + } + } + + private clearPausedForGrant(grant: Readonly): void { + for (const [id, identity] of this.pausedDeliveryByPty) { + if ( + identity.clientGeneration !== grant.clientGeneration || + identity.ownerGeneration !== grant.ownerGeneration + ) { + continue + } + this.pausedDeliveryByPty.delete(id) + this.setDeliveryPaused?.(id, false) + } + } +} diff --git a/src/relay/ssh-pty-source-credit-adapter.test.ts b/src/relay/ssh-pty-source-credit-adapter.test.ts new file mode 100644 index 00000000000..f738395a154 --- /dev/null +++ b/src/relay/ssh-pty-source-credit-adapter.test.ts @@ -0,0 +1,133 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + PTY_CONSUMER_OWNER_GRACE_MS, + PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + type PtyConsumerSessionGrant +} from '../shared/pty-consumer-session' +import { SshPtySourceCreditAdapter } from './ssh-pty-source-credit-adapter' + +function ownerGrant( + clientGeneration: number, + ownerGeneration: number +): Readonly { + return Object.freeze({ + protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + serverBuildId: 'build-a', + clientGeneration, + role: 'session-owner', + ownerGeneration, + ownerLease: `lease-${ownerGeneration}`, + capabilities: { outputFlowControl: { version: 1 as const, windowSu: 8 } } + }) +} + +afterEach(() => vi.useRealTimers()) + +describe('SshPtySourceCreditAdapter cleanup', () => { + it('grace expiry cancels old-owner tokens without closing a rotated replacement', () => { + vi.useFakeTimers() + const publishCancellation = vi.fn() + const adapter = new SshPtySourceCreditAdapter(publishCancellation) + const oldGrant = ownerGrant(1, 1) + const replacementGrant = ownerGrant(2, 2) + const rotated = adapter.open(oldGrant, 'pty-1', 'incarnation-1')! + const expiring = adapter.open(oldGrant, 'pty-2', 'incarnation-2')! + adapter.retainOrCloseOnDetach(oldGrant) + + const replacement = adapter.rotate(rotated, replacementGrant, 0).identity + vi.advanceTimersByTime(PTY_CONSUMER_OWNER_GRACE_MS) + + expect(adapter.snapshot(expiring).state).toBe('closed') + expect(adapter.snapshot(replacement).state).toBe('active') + expect(publishCancellation).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryToken: expiring.deliveryToken, + reason: 'reconnect-grace-expired', + remainingStartSu: 0, + remainingEndSu: 0 + }) + ) + }) + + it('non-owner detach cleanup is exact to the client generation', () => { + const adapter = new SshPtySourceCreditAdapter() + const detached = adapter.open(ownerGrant(1, 1), 'pty-1', 'incarnation-1')! + const active = adapter.open(ownerGrant(2, 2), 'pty-2', 'incarnation-2')! + const subscriber: Readonly = Object.freeze({ + protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + serverBuildId: 'build-a', + clientGeneration: 1, + role: 'subscriber' + }) + + adapter.retainOrCloseOnDetach(subscriber) + + expect(adapter.snapshot(detached).state).toBe('closed') + expect(adapter.snapshot(active).state).toBe('active') + }) + + it('prunes tokens across many normal sealed exits', () => { + const adapter = new SshPtySourceCreditAdapter() + const grant = ownerGrant(1, 1) + + for (let index = 0; index < 300; index++) { + const identity = adapter.open(grant, `pty-${index}`, `incarnation-${index}`)! + adapter.seal(identity) + adapter.settleExit(identity, { ok: true }) + } + + expect(adapter.retentionSnapshot()).toEqual({ + deliveryTokens: 0, + graceTimers: 0, + sourceSu: 0, + dataBytes: 0, + spans: 0 + }) + }) + + it('disposes grace timers and exact retained tokens', () => { + vi.useFakeTimers() + const adapter = new SshPtySourceCreditAdapter() + const grant = ownerGrant(1, 1) + const first = adapter.open(grant, 'pty-1', 'incarnation-1')! + const second = adapter.open(grant, 'pty-2', 'incarnation-2')! + adapter.retainOrCloseOnDetach(grant) + expect(adapter.retentionSnapshot()).toEqual({ + deliveryTokens: 2, + graceTimers: 1, + sourceSu: 0, + dataBytes: 0, + spans: 0 + }) + + adapter.dispose() + + expect(adapter.retentionSnapshot()).toEqual({ + deliveryTokens: 0, + graceTimers: 0, + sourceSu: 0, + dataBytes: 0, + spans: 0 + }) + expect(adapter.snapshot(first)).toMatchObject({ state: 'closed', generationClosed: true }) + expect(adapter.snapshot(second)).toMatchObject({ state: 'closed', generationClosed: true }) + expect(() => adapter.open(grant, 'pty-late', 'incarnation-late')).toThrow('disposed') + }) + + it('returns the same bounded proof for duplicate token cancellation', () => { + const adapter = new SshPtySourceCreditAdapter() + const grant = ownerGrant(1, 1) + const identity = adapter.open(grant, 'pty-1', 'incarnation-1')! + const params = { + id: identity.id, + deliveryToken: identity.deliveryToken, + clientGeneration: identity.clientGeneration, + ownerGeneration: identity.ownerGeneration + } + + const first = adapter.cancel(params, grant) + const duplicate = adapter.cancel(params, grant) + + expect(duplicate).toEqual(first) + }) +}) diff --git a/src/relay/ssh-pty-source-credit-adapter.ts b/src/relay/ssh-pty-source-credit-adapter.ts new file mode 100644 index 00000000000..d551dec8142 --- /dev/null +++ b/src/relay/ssh-pty-source-credit-adapter.ts @@ -0,0 +1,323 @@ +import { randomUUID } from 'node:crypto' +import { + MAX_PTY_ACK_ENTRIES, + type PtySourceDeliveryCancellation, + type PtySourceDeliveryIdentity, + type PtySourceDeliverySnapshot, + type PtySourceSpan, + type PtySourceTransform +} from '../shared/pty-source-credit-contract' +import { + PTY_CONSUMER_OWNER_GRACE_MS, + type PtyConsumerSessionGrant +} from '../shared/pty-consumer-session' +import { + ptySourceCancellationResult, + RecentPtySourceCancellationIndex +} from './pty-source-cancellation-index' +import { ownedPtySourceDelivery } from './pty-source-delivery-ownership' +import { + RelayPtySourceCreditLedger, + type PtySourceSendReservation +} from './pty-source-credit-ledger' + +export class SshPtySourceCreditAdapter { + private readonly sourceCredit = new RelayPtySourceCreditLedger() + private readonly identityByToken = new Map() + private readonly graceTimers = new Map>() + private readonly recentCancellations = new RecentPtySourceCancellationIndex() + private readonly relayProviderGeneration = 1 + private disposed = false + + constructor( + private readonly publishCancellation?: (proof: PtySourceDeliveryCancellation) => void, + private readonly onCreditAvailable?: (id: string) => void + ) {} + + open( + grant: Readonly | null, + id: string, + ptyIncarnation: string, + checkpointSourceEndSu = 0 + ): PtySourceDeliveryIdentity | null { + if (this.disposed) { + throw new Error('SSH PTY source credit adapter is disposed') + } + const flow = grant?.capabilities?.outputFlowControl + if ( + !grant || + grant.role !== 'session-owner' || + !grant.ownerGeneration || + !flow || + !id || + !ptyIncarnation + ) { + return null + } + const identity = Object.freeze({ + id, + providerGeneration: this.relayProviderGeneration, + clientGeneration: grant.clientGeneration, + ownerGeneration: grant.ownerGeneration, + ptyIncarnation, + deliveryToken: randomUUID() + }) + this.sourceCredit.open(identity, flow.windowSu, checkpointSourceEndSu) + this.identityByToken.set(identity.deliveryToken, identity) + return identity + } + + rotate( + oldIdentity: PtySourceDeliveryIdentity, + grant: Readonly | null, + acceptedSourceEndSu: number + ) { + const flow = grant?.capabilities?.outputFlowControl + if (!grant || !flow || !grant.ownerGeneration || grant.role !== 'session-owner') { + throw new Error('PTY source delivery recovery requires the active negotiated owner') + } + const replacement = Object.freeze({ + ...oldIdentity, + clientGeneration: grant.clientGeneration, + ownerGeneration: grant.ownerGeneration, + deliveryToken: randomUUID() + }) + const rotation = this.sourceCredit.rotate( + oldIdentity, + replacement, + acceptedSourceEndSu, + flow.windowSu + ) + this.identityByToken.delete(oldIdentity.deliveryToken) + this.recentCancellations.remember(rotation.cancellation) + this.publishCancellation?.(rotation.cancellation) + this.identityByToken.set(replacement.deliveryToken, replacement) + this.clearGraceWhenSettled(oldIdentity.ownerGeneration) + return Object.freeze({ identity: replacement, ...rotation }) + } + + append( + identity: PtySourceDeliveryIdentity, + input: Readonly<{ + spanId: string + data: string + displayStart: number + displayEnd: number + splittable: boolean + transform: PtySourceTransform + }> + ): PtySourceSpan { + return this.sourceCredit.append(identity, input) + } + + reserveSend( + identity: PtySourceDeliveryIdentity, + maxSourceSu?: number + ): PtySourceSendReservation | null { + return this.sourceCredit.reserveNextSend(identity, maxSourceSu) + } + + commitSend(reservation: PtySourceSendReservation): void { + this.sourceCredit.commitSend(reservation) + } + + rollbackSend(reservation: PtySourceSendReservation): void { + this.sourceCredit.rollbackSend(reservation) + } + + seal(identity: PtySourceDeliveryIdentity): void { + this.sourceCredit.seal(identity) + } + + settleExit( + identity: PtySourceDeliveryIdentity, + result: { ok: true } | { ok: false; error: Error } + ): void { + this.sourceCredit.settleExitPublication(identity, result) + this.pruneClosed(identity.deliveryToken, identity) + } + + snapshot(identity: PtySourceDeliveryIdentity): PtySourceDeliverySnapshot { + return this.sourceCredit.snapshot(identity) + } + + acknowledge( + params: Record, + grant: Readonly | null + ): void { + const acknowledgements = params.acknowledgements + if ( + !grant?.capabilities?.outputFlowControl || + !Array.isArray(acknowledgements) || + acknowledgements.length > MAX_PTY_ACK_ENTRIES + ) { + return + } + for (const raw of acknowledgements) { + if (typeof raw !== 'object' || raw === null) { + continue + } + const candidate = raw as Record + const token = typeof candidate.deliveryToken === 'string' ? candidate.deliveryToken : '' + const identity = token.length > 0 ? this.identityByToken.get(token) : undefined + if (!identity || identity.clientGeneration !== grant.clientGeneration) { + continue + } + try { + const result = this.sourceCredit.acknowledge(identity, { + id: String(candidate.id ?? ''), + clientGeneration: Number(candidate.clientGeneration), + ownerGeneration: Number(candidate.ownerGeneration), + deliveryToken: String(candidate.deliveryToken), + creditedEndSu: Number(candidate.creditedEndSu) + }) + if (result === 'advanced') { + this.onCreditAvailable?.(identity.id) + } + this.pruneClosed(token, identity) + } catch { + /* Invalid and stale cumulative ACKs never mutate credit. */ + } + } + } + + cancel( + params: Record, + grant: Readonly | null + ): Readonly<{ canceled: true; sentEndSu: number; creditedEndSu: number }> { + const token = typeof params.deliveryToken === 'string' ? params.deliveryToken : '' + const identity = this.identityByToken.get(token) + const recent = this.recentCancellations.owned(token, params, grant) + if (!identity && recent) { + return ptySourceCancellationResult(recent) + } + if ( + !grant?.capabilities?.outputFlowControl || + !identity || + identity.clientGeneration !== grant.clientGeneration || + params.id !== identity.id || + Number(params.clientGeneration) !== identity.clientGeneration || + Number(params.ownerGeneration) !== identity.ownerGeneration + ) { + throw new Error('Unknown or stale PTY source delivery cancellation') + } + const proof = this.sourceCredit.cancel(identity, 'client-request') + this.identityByToken.delete(token) + this.recentCancellations.remember(proof) + this.clearGraceWhenSettled(identity.ownerGeneration) + return ptySourceCancellationResult(proof) + } + + retainOrCloseOnDetach(grant: Readonly): void { + if (grant.role === 'session-owner' && grant.capabilities?.outputFlowControl) { + if (this.hasOwnerDeliveries(grant.ownerGeneration!)) { + this.scheduleGraceExpiry(grant.ownerGeneration!) + } + return + } + this.closeClientDeliveries(grant.clientGeneration) + } + + retentionSnapshot() { + return Object.freeze({ + deliveryTokens: this.identityByToken.size, + graceTimers: this.graceTimers.size, + ...this.sourceCredit.retentionSnapshot() + }) + } + + ownsDelivery( + token: string, + grant: Readonly, + id: string + ): PtySourceDeliveryIdentity | null { + return ownedPtySourceDelivery(this.identityByToken.get(token), grant, id) + } + + cancelIdentity(identity: PtySourceDeliveryIdentity, reason: string): void { + const token = identity.deliveryToken + if (this.identityByToken.get(token) !== identity) { + return + } + this.cancelExact(token, identity, reason) + } + + dispose(): void { + if (this.disposed) { + return + } + this.disposed = true + for (const timer of this.graceTimers.values()) { + clearTimeout(timer) + } + this.graceTimers.clear() + this.sourceCredit.closeGeneration(this.relayProviderGeneration) + this.identityByToken.clear() + this.recentCancellations.clear() + } + + private closeClientDeliveries(clientGeneration: number): void { + for (const [token, identity] of this.identityByToken) { + if (identity.clientGeneration !== clientGeneration) { + continue + } + this.cancelExact(token, identity, 'client-detached') + } + } + + private scheduleGraceExpiry(ownerGeneration: number): void { + this.clearGraceTimer(ownerGeneration) + const timer = setTimeout(() => { + this.graceTimers.delete(ownerGeneration) + for (const [token, identity] of this.identityByToken) { + if (identity.ownerGeneration !== ownerGeneration) { + continue + } + this.cancelExact(token, identity, 'reconnect-grace-expired') + } + }, PTY_CONSUMER_OWNER_GRACE_MS) + timer.unref?.() + this.graceTimers.set(ownerGeneration, timer) + } + + private cancelExact(token: string, identity: PtySourceDeliveryIdentity, reason: string): void { + if (this.sourceCredit.snapshot(identity).state !== 'closed') { + const proof = this.sourceCredit.cancel(identity, reason) + this.recentCancellations.remember(proof) + this.publishCancellation?.(proof) + } + if (this.identityByToken.get(token) === identity) { + this.identityByToken.delete(token) + } + this.clearGraceWhenSettled(identity.ownerGeneration) + } + + private pruneClosed(token: string, identity: PtySourceDeliveryIdentity): void { + if ( + this.sourceCredit.snapshot(identity).state === 'closed' && + this.identityByToken.get(token) === identity + ) { + this.identityByToken.delete(token) + this.clearGraceWhenSettled(identity.ownerGeneration) + } + } + + private hasOwnerDeliveries = (ownerGeneration: number): boolean => + Array.from(this.identityByToken.values()).some( + (identity) => identity.ownerGeneration === ownerGeneration + ) + + private clearGraceWhenSettled(ownerGeneration: number): void { + if (!this.hasOwnerDeliveries(ownerGeneration)) { + this.clearGraceTimer(ownerGeneration) + } + } + + private clearGraceTimer(ownerGeneration: number): void { + const timer = this.graceTimers.get(ownerGeneration) + if (timer) { + clearTimeout(timer) + this.graceTimers.delete(ownerGeneration) + } + } +} diff --git a/src/relay/wsl-agent-hook-relay.ts b/src/relay/wsl-agent-hook-relay.ts index bda345806b9..534c388e07c 100644 --- a/src/relay/wsl-agent-hook-relay.ts +++ b/src/relay/wsl-agent-hook-relay.ts @@ -38,12 +38,26 @@ async function main(): Promise { } let stdoutAlive = true - const dispatcher = new RelayDispatcher((data) => { - if (!stdoutAlive) { - return + const dispatcher = new RelayDispatcher( + (data, onSettled) => { + if (!stdoutAlive) { + onSettled({ ok: false, error: new Error('WSL relay stdout is closed') }) + return false + } + return process.stdout.write(data, (error) => { + onSettled(error ? { ok: false, error } : { ok: true }) + }) + }, + { + supportsWriteCallback: true, + writableLength: () => process.stdout.writableLength, + writableHighWaterMark: () => process.stdout.writableHighWaterMark, + waitWriteDrain: (callback) => { + process.stdout.once('drain', callback) + return () => process.stdout.off('drain', callback) + } } - return process.stdout.write(data) - }) + ) // Why: restart-stable instance key keeps the endpoint file at one path // across app restarts so surviving agents re-coordinate off its rewrite. @@ -120,7 +134,7 @@ async function main(): Promise { // Signal readiness — the host watches for this exact string before // sending framed data (same contract as the SSH relay). - process.stdout.write(RELAY_SENTINEL) + dispatcher.writePrimaryBytes(Buffer.from(RELAY_SENTINEL)) } void main() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts index 8a6cc5b611e..08044fed354 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts @@ -225,6 +225,7 @@ describe('createRemoteRuntimePtyTransport', () => { await vi.waitFor(() => expect(latestSubscribePayload().capabilities).toEqual({ ackOutput: 1, + ackOutputSourceRanges: 1, desktopViewportClaims: 1 }) ) diff --git a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts index 0c13d2d20bf..3e9d67fbc05 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts @@ -26,7 +26,12 @@ type RuntimeEnvironmentSubscriptionHandle = { type TerminalMultiplexEvent = | { type: 'ready' } - | { type: 'subscribed'; streamId: number } + | { + type: 'subscribed' + streamId: number + streamGeneration?: string + capabilities?: { ackOutputSourceRanges?: 1 } + } | { type: 'end'; streamId: number } | { type: 'error'; streamId: number; message?: string } | { @@ -81,6 +86,9 @@ type RemoteRuntimeMultiplexedTerminalState = { callbacks: RemoteRuntimeMultiplexedTerminalCallbacks subscriptionRequested: boolean acknowledgeOutput: boolean + acknowledgeOutputSourceRanges: boolean + streamGeneration: string | null + sourceAckedEndByte: number heldAckBytes: number pendingAckBytes: number ackFlushTimer: ReturnType | null @@ -251,7 +259,10 @@ class RemoteRuntimeTerminalMultiplexer { terminal: args.terminal, callbacks: args.callbacks, subscriptionRequested: false, - acknowledgeOutput: args.client.type === 'desktop', + acknowledgeOutput: true, + acknowledgeOutputSourceRanges: false, + streamGeneration: null, + sourceAckedEndByte: 0, heldAckBytes: 0, pendingAckBytes: 0, ackFlushTimer: null, @@ -323,8 +334,11 @@ class RemoteRuntimeTerminalMultiplexer { terminal: args.terminal, client: args.client, viewport: args.viewport, - capabilities: - args.client.type === 'desktop' ? { ackOutput: 1, desktopViewportClaims: 1 } : undefined + capabilities: { + ackOutput: 1, + ackOutputSourceRanges: 1, + ...(args.client.type === 'desktop' ? { desktopViewportClaims: 1 } : {}) + } }) ) if (!sent) { @@ -437,7 +451,20 @@ class RemoteRuntimeTerminalMultiplexer { if (!stream) { return } - if (event.type === 'end') { + if (event.type === 'subscribed') { + const capabilities = + typeof event.capabilities === 'object' && event.capabilities !== null + ? (event.capabilities as { ackOutputSourceRanges?: unknown }) + : null + if ( + capabilities?.ackOutputSourceRanges === 1 && + typeof event.streamGeneration === 'string' && + event.streamGeneration.length > 0 + ) { + stream.acknowledgeOutputSourceRanges = true + stream.streamGeneration = event.streamGeneration + } + } else if (event.type === 'end') { discardOutputAcknowledgements(stream) clearSnapshot(stream) clearResyncTimer(stream) @@ -874,6 +901,21 @@ class RemoteRuntimeTerminalMultiplexer { } private acknowledgeOutput(stream: RemoteRuntimeMultiplexedTerminalState, bytes: number): boolean { + if (stream.acknowledgeOutputSourceRanges && stream.streamGeneration) { + const ackedEndByte = stream.sourceAckedEndByte + bytes + const sent = this.sendFrame( + stream.streamId, + TerminalStreamOpcode.Ack, + encodeTerminalStreamJson({ + streamGeneration: stream.streamGeneration, + ackedEndByte + }) + ) + if (sent) { + stream.sourceAckedEndByte = ackedEndByte + } + return sent + } return this.sendFrame( stream.streamId, TerminalStreamOpcode.Ack, diff --git a/src/renderer/src/runtime/runtime-terminal-stream.test.ts b/src/renderer/src/runtime/runtime-terminal-stream.test.ts index a990b146e4d..be485d0ef2f 100644 --- a/src/renderer/src/runtime/runtime-terminal-stream.test.ts +++ b/src/renderer/src/runtime/runtime-terminal-stream.test.ts @@ -102,11 +102,16 @@ describe('remote runtime terminal data subscriptions', () => { subscribeFrame && decodeTerminalStreamJson<{ streamId: number - capabilities?: { ackOutput?: 1; desktopViewportClaims?: 1 } + capabilities?: { + ackOutput?: 1 + ackOutputSourceRanges?: 1 + desktopViewportClaims?: 1 + } }>(subscribeFrame.payload) expect(subscribePayload?.streamId).toEqual(expect.any(Number)) expect(subscribePayload?.capabilities).toEqual({ ackOutput: 1, + ackOutputSourceRanges: 1, desktopViewportClaims: 1 }) @@ -140,6 +145,49 @@ describe('remote runtime terminal data subscriptions', () => { expect(_getRemoteRuntimeTerminalMultiplexerCountForTest()).toBe(0) }) + it('echoes the opaque stream generation in cumulative source-range ACKs', async () => { + const dispose = await subscribeToRuntimeTerminalData( + { activeRuntimeEnvironmentId: 'env-fallback' }, + 'remote:env-1@@terminal-1', + 'watcher-1', + vi.fn() + ) + await vi.waitFor(() => expect(sendBinary).toHaveBeenCalled()) + const subscribeFrame = decodeTerminalStreamFrame(sendBinary.mock.calls[0][0]) + const subscribePayload = + subscribeFrame && decodeTerminalStreamJson<{ streamId: number }>(subscribeFrame.payload) + callbacks?.onResponse({ + ok: true, + result: { + type: 'subscribed', + streamId: subscribePayload!.streamId, + streamGeneration: 'opaque-generation', + capabilities: { ackOutputSourceRanges: 1 } + } + }) + + callbacks?.onBinary?.( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Output, + streamId: subscribePayload!.streamId, + seq: 4, + payload: encodeTerminalStreamText('live') + }) + ) + + await vi.waitFor(() => { + const ack = sendBinary.mock.calls + .slice(1) + .map((call) => decodeTerminalStreamFrame(call[0])) + .find((frame) => frame?.opcode === TerminalStreamOpcode.Ack) + expect(ack && decodeTerminalStreamJson(ack.payload)).toEqual({ + streamGeneration: 'opaque-generation', + ackedEndByte: 4 + }) + }) + dispose() + }) + it('can start at the live tail without replaying the initial snapshot', async () => { const watcher = vi.fn() const subscription = subscribeToRuntimeTerminalData( diff --git a/src/shared/pty-consumer-session-contract.ts b/src/shared/pty-consumer-session-contract.ts new file mode 100644 index 00000000000..661138896cd --- /dev/null +++ b/src/shared/pty-consumer-session-contract.ts @@ -0,0 +1,59 @@ +export const PTY_CONSUMER_SESSION_PROTOCOL_VERSION = 1 +export const PTY_CONSUMER_OWNER_GRACE_MS = 30_000 +export const PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR = -32041 + +export type PtyConsumerRole = 'session-owner' | 'subscriber' + +export type PtyConsumerSessionHello = { + clientInstanceId: string + requestedRole: PtyConsumerRole + resume?: { + ownerGeneration: number + ownerLease: string + } + capabilities?: { + outputFlowControl?: { + versions: number[] + requestedWindowSu: number + } + } +} + +export type PtyConsumerSessionGrant = { + protocolVersion: typeof PTY_CONSUMER_SESSION_PROTOCOL_VERSION + serverBuildId: string + clientGeneration: number + role: PtyConsumerRole + ownerGeneration?: number + ownerLease?: string + capabilities?: { + outputFlowControl?: { + version: 1 + windowSu: number + } + } +} + +export type PtyConsumerAuthentication = { + connectionId: string + principal: string + authenticated: boolean + allowSessionOwner: boolean +} + +export type PtyConsumerSessionAdmission = { + grant: Readonly + commitPublication: () => void + rollbackPublication: () => void +} + +export type PtyConsumerSessionOptions = { + serverBuildId: string + outputFlowControl?: { + versions: readonly number[] + maxWindowSu: number + } + ownerGraceMs?: number + now?: () => number + createLease?: () => string +} diff --git a/src/shared/pty-consumer-session.test.ts b/src/shared/pty-consumer-session.test.ts new file mode 100644 index 00000000000..07f9f7f9bf9 --- /dev/null +++ b/src/shared/pty-consumer-session.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, it } from 'vitest' +import { + PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, + PtyConsumerSession, + type PtyConsumerAuthentication, + type PtyConsumerSessionHello +} from './pty-consumer-session' + +function auth( + connectionId: string, + overrides: Partial = {} +): PtyConsumerAuthentication { + return { + connectionId, + principal: 'desktop', + authenticated: true, + allowSessionOwner: true, + ...overrides + } +} + +function ownerHello(overrides: Partial = {}): PtyConsumerSessionHello { + return { + clientInstanceId: 'client-a', + requestedRole: 'session-owner', + ...overrides + } +} + +function createSession(options: { now?: () => number } = {}): PtyConsumerSession { + let lease = 0 + return new PtyConsumerSession({ + serverBuildId: 'relay-build', + createLease: () => `lease-${++lease}`, + ownerGraceMs: 30_000, + ...options + }) +} + +describe('PtyConsumerSession', () => { + it('types a resume against a fresh relay without weakening other owner refusals', () => { + const session = createSession() + + expect(() => + session.admit( + ownerHello({ resume: { ownerGeneration: 1, ownerLease: 'stale' } }), + auth('connection-1') + ) + ).toThrow( + expect.objectContaining({ + code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, + message: expect.stringContaining('stale') + }) + ) + }) + + it('activates an authenticated owner only after its publication fence', () => { + const session = createSession() + const first = session.admit(ownerHello(), auth('connection-1')) + const competitor = session.admit( + ownerHello({ clientInstanceId: 'client-b' }), + auth('connection-2', { principal: 'other' }) + ) + + expect(first.grant).toMatchObject({ + clientGeneration: 1, + role: 'session-owner', + ownerGeneration: 1, + ownerLease: 'lease-1' + }) + expect(competitor.grant).toMatchObject({ clientGeneration: 2, role: 'subscriber' }) + first.commitPublication() + }) + + it('rolls back an unpublished owner without consuming authority', () => { + const session = createSession() + session.admit(ownerHello(), auth('failed')).rollbackPublication() + + const retry = session.admit(ownerHello(), auth('retry')) + expect(retry.grant).toMatchObject({ + role: 'session-owner', + ownerGeneration: 2, + ownerLease: 'lease-2' + }) + }) + + it('returns the same generation and lease for duplicate opens on one connection', () => { + const session = createSession() + const first = session.admit(ownerHello(), auth('connection-1')) + const duplicate = session.admit(ownerHello(), auth('connection-1')) + + expect(duplicate.grant).toBe(first.grant) + first.commitPublication() + duplicate.commitPublication() + }) + + it('rejects a second, different open on one connection', () => { + const session = createSession() + session.admit(ownerHello(), auth('connection-1')) + + expect(() => + session.admit(ownerHello({ requestedRole: 'subscriber' }), auth('connection-1')) + ).toThrow('only once') + }) + + it('cannot self-promote an authenticated but owner-ineligible principal', () => { + const session = createSession() + const admission = session.admit( + ownerHello(), + auth('connection-1', { allowSessionOwner: false }) + ) + + expect(admission.grant.role).toBe('subscriber') + expect(admission.grant.ownerLease).toBeUndefined() + }) + + it('rejects an unauthenticated transport', () => { + const session = createSession() + expect(() => + session.admit(ownerHello(), auth('connection-1', { authenticated: false })) + ).toThrow('authentication required') + }) + + it('rotates the lease and increments owner generation on valid recovery', () => { + const session = createSession() + const first = session.admit(ownerHello(), auth('connection-1')) + first.commitPublication() + session.close('connection-1') + + const recovered = session.admit( + ownerHello({ + resume: { + ownerGeneration: first.grant.ownerGeneration!, + ownerLease: first.grant.ownerLease! + } + }), + auth('connection-2') + ) + recovered.commitPublication() + + expect(recovered.grant).toMatchObject({ + role: 'session-owner', + ownerGeneration: 2, + ownerLease: 'lease-2' + }) + }) + + it('rejects recovery while the current owner is still active', () => { + const session = createSession() + const first = session.admit(ownerHello(), auth('connection-1')) + first.commitPublication() + + expect(() => + session.admit( + ownerHello({ + resume: { + ownerGeneration: first.grant.ownerGeneration!, + ownerLease: first.grant.ownerLease! + } + }), + auth('connection-2') + ) + ).toThrow('Active owner') + }) + + it('types mismatched recovery without disturbing principal or lease ownership', () => { + const session = createSession() + const first = session.admit(ownerHello(), auth('connection-1')) + first.commitPublication() + session.close('connection-1') + + expect(() => + session.admit( + ownerHello({ resume: { ownerGeneration: 1, ownerLease: 'lease-1' } }), + auth('connection-2', { principal: 'stale-desktop' }) + ) + ).toThrow( + expect.objectContaining({ + code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, + message: expect.stringContaining('principal') + }) + ) + expect(() => + session.admit( + ownerHello({ resume: { ownerGeneration: 1, ownerLease: 'wrong' } }), + auth('connection-3') + ) + ).toThrow(expect.objectContaining({ code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR })) + + const staleFresh = session.admit( + ownerHello(), + auth('connection-2', { principal: 'stale-desktop' }) + ) + expect(staleFresh.grant).toMatchObject({ role: 'subscriber' }) + expect(staleFresh.grant.ownerLease).toBeUndefined() + + const recovered = session.admit( + ownerHello({ resume: { ownerGeneration: 1, ownerLease: 'lease-1' } }), + auth('connection-4') + ) + expect(recovered.grant).toMatchObject({ + role: 'session-owner', + ownerGeneration: 2, + ownerLease: 'lease-2' + }) + }) + + it('elects a new owner after disconnected-owner grace expires', () => { + let now = 10 + const session = createSession({ now: () => now }) + const first = session.admit(ownerHello(), auth('connection-1')) + first.commitPublication() + session.close('connection-1') + now += 30_000 + + const next = session.admit( + ownerHello({ clientInstanceId: 'client-b' }), + auth('connection-2', { principal: 'other' }) + ) + expect(next.grant).toMatchObject({ role: 'session-owner', ownerGeneration: 2 }) + }) + + it('intersects V1 capability and clamps its source-unit window', () => { + const session = new PtyConsumerSession({ + serverBuildId: 'relay-build', + outputFlowControl: { versions: [1], maxWindowSu: 64 }, + createLease: () => 'lease' + }) + const admission = session.admit( + ownerHello({ + capabilities: { + outputFlowControl: { versions: [1, 2], requestedWindowSu: 128 } + } + }), + auth('connection-1') + ) + + expect(admission.grant.capabilities?.outputFlowControl).toEqual({ + version: 1, + windowSu: 64 + }) + }) + + it('makes token-free bounded legacy an explicit capability omission', () => { + const session = createSession() + const admission = session.admit(ownerHello(), auth('connection-1')) + + expect(admission.grant.capabilities).toBeUndefined() + expect(admission.grant).not.toHaveProperty('deliveryToken') + }) + + it('bounds capability offers before fingerprinting them', () => { + const session = createSession() + expect(() => + session.admit( + ownerHello({ + capabilities: { + outputFlowControl: { + versions: Array.from({ length: 9 }, (_, index) => index + 1), + requestedWindowSu: 64 + } + } + }), + auth('connection-1') + ) + ).toThrow('versions') + }) + + it('rejects invalid server windows and owner grace', () => { + expect( + () => + new PtyConsumerSession({ + serverBuildId: 'build', + outputFlowControl: { versions: [1], maxWindowSu: 0 } + }) + ).toThrow('support') + expect( + () => new PtyConsumerSession({ serverBuildId: 'build', ownerGraceMs: Number.MAX_VALUE }) + ).toThrow('ownerGraceMs') + }) +}) diff --git a/src/shared/pty-consumer-session.ts b/src/shared/pty-consumer-session.ts new file mode 100644 index 00000000000..953b77c3a8d --- /dev/null +++ b/src/shared/pty-consumer-session.ts @@ -0,0 +1,321 @@ +import { randomUUID } from 'node:crypto' +import { + PTY_CONSUMER_OWNER_GRACE_MS, + PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR, + type PtyConsumerAuthentication, + type PtyConsumerSessionAdmission, + type PtyConsumerSessionGrant, + type PtyConsumerSessionHello, + type PtyConsumerSessionOptions +} from './pty-consumer-session-contract' + +export * from './pty-consumer-session-contract' + +const MAX_CAPABILITY_VERSIONS = 8 + +type ClientRecord = { + fingerprint: string + principal: string + clientInstanceId: string + grant: Readonly + state: 'pending' | 'active' + publicationState: 'pending' | 'committed' | 'rolled-back' +} + +type OwnerRecord = { + connectionId: string + principal: string + clientInstanceId: string + generation: number + lease: string + state: 'pending' | 'active' | 'disconnected' + disconnectedAt?: number + replaces?: OwnerRecord +} + +function assertNonEmptyString(value: unknown, name: string): asserts value is string { + if (typeof value !== 'string' || value.length === 0 || value.length > 512) { + throw new Error(`${name} must be a non-empty string of at most 512 characters`) + } +} + +function validateHello(hello: PtyConsumerSessionHello): void { + assertNonEmptyString(hello.clientInstanceId, 'clientInstanceId') + if (hello.requestedRole !== 'session-owner' && hello.requestedRole !== 'subscriber') { + throw new Error('requestedRole must be session-owner or subscriber') + } + if (hello.resume) { + if (!Number.isSafeInteger(hello.resume.ownerGeneration) || hello.resume.ownerGeneration <= 0) { + throw new Error('resume.ownerGeneration must be a positive safe integer') + } + assertNonEmptyString(hello.resume.ownerLease, 'resume.ownerLease') + } + const flow = hello.capabilities?.outputFlowControl + if (!flow) { + return + } + if ( + !Array.isArray(flow.versions) || + flow.versions.length > MAX_CAPABILITY_VERSIONS || + flow.versions.some((version) => !Number.isSafeInteger(version) || version <= 0) + ) { + throw new Error('outputFlowControl.versions must contain positive safe integers') + } + if (!Number.isSafeInteger(flow.requestedWindowSu) || flow.requestedWindowSu <= 0) { + throw new Error('outputFlowControl.requestedWindowSu must be a positive safe integer') + } +} + +function helloFingerprint(hello: PtyConsumerSessionHello): string { + const flow = hello.capabilities?.outputFlowControl + return JSON.stringify({ + clientInstanceId: hello.clientInstanceId, + requestedRole: hello.requestedRole, + resume: hello.resume, + outputFlowControl: flow + ? { + versions: [...flow.versions].sort((a, b) => a - b), + requestedWindowSu: flow.requestedWindowSu + } + : undefined + }) +} + +export class PtyConsumerSession { + private readonly clients = new Map() + private readonly now: () => number + private readonly createLease: () => string + private readonly ownerGraceMs: number + private nextClientGeneration = 1 + private nextOwnerGeneration = 1 + private owner: OwnerRecord | null = null + + constructor(private readonly options: PtyConsumerSessionOptions) { + assertNonEmptyString(options.serverBuildId, 'serverBuildId') + if ( + options.outputFlowControl && + (!Number.isSafeInteger(options.outputFlowControl.maxWindowSu) || + options.outputFlowControl.maxWindowSu <= 0 || + options.outputFlowControl.versions.length > MAX_CAPABILITY_VERSIONS || + options.outputFlowControl.versions.some( + (version) => !Number.isSafeInteger(version) || version <= 0 + )) + ) { + throw new Error('outputFlowControl support is invalid') + } + if ( + options.ownerGraceMs !== undefined && + (!Number.isSafeInteger(options.ownerGraceMs) || options.ownerGraceMs < 0) + ) { + throw new Error('ownerGraceMs must be a non-negative safe integer') + } + this.now = options.now ?? Date.now + this.createLease = options.createLease ?? randomUUID + this.ownerGraceMs = options.ownerGraceMs ?? PTY_CONSUMER_OWNER_GRACE_MS + } + + admit( + hello: PtyConsumerSessionHello, + authentication: PtyConsumerAuthentication + ): PtyConsumerSessionAdmission { + validateHello(hello) + assertNonEmptyString(authentication.connectionId, 'connectionId') + assertNonEmptyString(authentication.principal, 'principal') + if (!authentication.authenticated) { + throw new Error('PTY consumer authentication required') + } + this.expireOwner() + + const fingerprint = helloFingerprint(hello) + const duplicate = this.clients.get(authentication.connectionId) + if (duplicate) { + if ( + duplicate.fingerprint !== fingerprint || + duplicate.principal !== authentication.principal + ) { + throw new Error('pty.openClient may be used only once per transport connection') + } + return this.admissionFor(duplicate) + } + + const owner = this.selectOwner(hello, authentication) + const grant = Object.freeze({ + protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION, + serverBuildId: this.options.serverBuildId, + clientGeneration: this.nextClientGeneration++, + role: owner ? ('session-owner' as const) : ('subscriber' as const), + ...(owner ? { ownerGeneration: owner.generation, ownerLease: owner.lease } : {}), + ...this.intersectCapabilities(hello) + }) + const client: ClientRecord = { + fingerprint, + principal: authentication.principal, + clientInstanceId: hello.clientInstanceId, + grant, + state: 'pending', + publicationState: 'pending' + } + this.clients.set(authentication.connectionId, client) + if (owner) { + this.owner = owner + } + return this.admissionFor(client) + } + + close(connectionId: string): void { + const client = this.clients.get(connectionId) + if (!client) { + return + } + this.clients.delete(connectionId) + if (this.owner?.connectionId !== connectionId) { + return + } + if (this.owner.state === 'pending') { + this.owner = this.owner.replaces ?? null + return + } + this.owner = { + ...this.owner, + state: 'disconnected', + disconnectedAt: this.now() + } + } + + sweepExpired(): void { + this.expireOwner() + } + + activeGrant(connectionId: string): Readonly | null { + const client = this.clients.get(connectionId) + return client?.state === 'active' ? client.grant : null + } + + private admissionFor(client: ClientRecord): PtyConsumerSessionAdmission { + return { + grant: client.grant, + commitPublication: () => { + if (client.publicationState !== 'pending') { + return + } + client.publicationState = 'committed' + if (client.state !== 'pending') { + return + } + client.state = 'active' + const owner = this.owner + if (owner?.connectionId === this.connectionIdFor(client) && owner.state === 'pending') { + this.owner = { ...owner, state: 'active', replaces: undefined } + } + }, + rollbackPublication: () => { + if (client.publicationState !== 'pending') { + return + } + client.publicationState = 'rolled-back' + if (client.state !== 'pending') { + return + } + const connectionId = this.connectionIdFor(client) + this.clients.delete(connectionId) + if (this.owner?.connectionId === connectionId && this.owner.state === 'pending') { + this.owner = this.owner.replaces ?? null + } + } + } + } + + private connectionIdFor(client: ClientRecord): string { + for (const [connectionId, candidate] of this.clients) { + if (candidate === client) { + return connectionId + } + } + return '' + } + + private selectOwner( + hello: PtyConsumerSessionHello, + authentication: PtyConsumerAuthentication + ): OwnerRecord | null { + if (hello.requestedRole !== 'session-owner' || !authentication.allowSessionOwner) { + return null + } + const current = this.owner + if (!current) { + if (hello.resume) { + throw Object.assign(new Error('Owner recovery lease is stale'), { + code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR + }) + } + return this.newOwner(hello, authentication, null) + } + if (!hello.resume) { + return null + } + const recoveryMatches = + hello.resume.ownerGeneration === current.generation && + hello.resume.ownerLease === current.lease && + hello.clientInstanceId === current.clientInstanceId && + authentication.principal === current.principal + if (!recoveryMatches) { + throw Object.assign( + new Error('Owner recovery lease is stale or belongs to another principal'), + { code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } + ) + } + if (current.state === 'pending') { + throw new Error('Owner grant publication is still pending') + } + if (current.state !== 'disconnected') { + throw new Error('Active owner cannot be replaced by recovery') + } + return this.newOwner(hello, authentication, current) + } + + private newOwner( + hello: PtyConsumerSessionHello, + authentication: PtyConsumerAuthentication, + replaces: OwnerRecord | null + ): OwnerRecord { + const lease = this.createLease() + assertNonEmptyString(lease, 'ownerLease') + return { + connectionId: authentication.connectionId, + principal: authentication.principal, + clientInstanceId: hello.clientInstanceId, + generation: this.nextOwnerGeneration++, + lease, + state: 'pending', + ...(replaces ? { replaces } : {}) + } + } + + private intersectCapabilities( + hello: PtyConsumerSessionHello + ): Pick { + const offer = hello.capabilities?.outputFlowControl + const support = this.options.outputFlowControl + if (!offer || !support || !offer.versions.includes(1) || !support.versions.includes(1)) { + return {} + } + return { + capabilities: { + outputFlowControl: { + version: 1, + windowSu: Math.min(offer.requestedWindowSu, support.maxWindowSu) + } + } + } + } + + private expireOwner(): void { + if ( + this.owner?.state === 'disconnected' && + this.now() - (this.owner.disconnectedAt ?? this.now()) >= this.ownerGraceMs + ) { + this.owner = null + } + } +} diff --git a/src/shared/pty-retained-string-memory.ts b/src/shared/pty-retained-string-memory.ts new file mode 100644 index 00000000000..a6d1143dbb1 --- /dev/null +++ b/src/shared/pty-retained-string-memory.ts @@ -0,0 +1,5 @@ +export const PTY_RETAINED_RECORD_BYTES = 128 + +export function chargedPtyRetainedStringBytes(value: string): number { + return Math.max(Buffer.byteLength(value, 'utf8'), 2 * value.length) + PTY_RETAINED_RECORD_BYTES +} diff --git a/src/shared/pty-source-credit-contract.ts b/src/shared/pty-source-credit-contract.ts new file mode 100644 index 00000000000..a97b66ec4a2 --- /dev/null +++ b/src/shared/pty-source-credit-contract.ts @@ -0,0 +1,90 @@ +export const PTY_SOURCE_CREDIT_VERSION = 1 +export const DEFAULT_PTY_SOURCE_WINDOW_SU = 256 * 1024 +export const MAX_PTY_ACK_ENTRIES = 64 + +export type PtySourceDeliveryIdentity = Readonly<{ + id: string + providerGeneration: number + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string +}> + +export type PtySourceTransform = Readonly<{ + transformed: boolean + rawLengthSu: number + scalarSafe: boolean +}> + +export type PtySourceSpan = PtySourceDeliveryIdentity & + Readonly<{ + spanId: string + sourceStartSu: number + sourceEndSu: number + displayStart: number + displayEnd: number + data: string + splittable?: boolean + indivisible?: boolean + transform: PtySourceTransform + }> + +export type PtySourceCreditAck = Readonly<{ + id: string + clientGeneration: number + ownerGeneration: number + deliveryToken: string + creditedEndSu: number +}> + +export type PtySourceCreditAckBatch = Readonly<{ + acknowledgements: readonly PtySourceCreditAck[] +}> + +export type PtySourceDeliveryCancellation = PtySourceDeliveryIdentity & + Readonly<{ + reason: string + sentEndSu: number + creditedEndSu: number + remainingStartSu: number + remainingEndSu: number + replacementDeliveryToken?: string + }> + +export type PtySourceDeliverySnapshot = PtySourceDeliveryIdentity & + Readonly<{ + state: 'active' | 'sealed-unsettled' | 'closing' | 'closed' + windowSu: number + receivedEndSu: number + sentEndSu: number + creditedEndSu: number + exitPublished: boolean + generationClosed: boolean + }> + +export function ptySourceDeliveryKey( + identity: Pick +): string { + return `${identity.providerGeneration}\0${identity.deliveryToken}` +} + +export function samePtySourceDelivery( + left: PtySourceDeliveryIdentity, + right: PtySourceDeliveryIdentity +): boolean { + return ( + left.id === right.id && + left.providerGeneration === right.providerGeneration && + left.clientGeneration === right.clientGeneration && + left.ownerGeneration === right.ownerGeneration && + left.ptyIncarnation === right.ptyIncarnation && + left.deliveryToken === right.deliveryToken + ) +} + +export function ptySourceSpanIsSplittable( + span: Pick +): boolean { + return span.splittable ?? (span.indivisible !== undefined ? !span.indivisible : false) +} diff --git a/src/shared/pty-source-credit-validation.ts b/src/shared/pty-source-credit-validation.ts new file mode 100644 index 00000000000..9c47aa25f90 --- /dev/null +++ b/src/shared/pty-source-credit-validation.ts @@ -0,0 +1,63 @@ +import type { + PtySourceCreditAck, + PtySourceDeliveryIdentity, + PtySourceSpan +} from './pty-source-credit-contract' + +export function assertPositiveSafeInteger(value: number, name: string): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive safe integer`) + } +} + +export function assertNonNegativeSafeInteger(value: number, name: string): void { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative safe integer`) + } +} + +export function assertPtySourceIdentity(identity: PtySourceDeliveryIdentity): void { + if (!identity.id || !identity.ptyIncarnation || !identity.deliveryToken) { + throw new Error('PTY source delivery identity is incomplete') + } + assertPositiveSafeInteger(identity.providerGeneration, 'providerGeneration') + assertPositiveSafeInteger(identity.clientGeneration, 'clientGeneration') + assertPositiveSafeInteger(identity.ownerGeneration, 'ownerGeneration') +} + +export function assertPtySourceSpan(span: PtySourceSpan): void { + assertPtySourceIdentity(span) + if (!span.spanId) { + throw new Error('spanId is required') + } + assertNonNegativeSafeInteger(span.sourceStartSu, 'sourceStartSu') + assertNonNegativeSafeInteger(span.sourceEndSu, 'sourceEndSu') + assertNonNegativeSafeInteger(span.displayStart, 'displayStart') + assertNonNegativeSafeInteger(span.displayEnd, 'displayEnd') + assertNonNegativeSafeInteger(span.transform.rawLengthSu, 'rawLengthSu') + if (span.sourceEndSu < span.sourceStartSu || span.displayEnd < span.displayStart) { + throw new Error('PTY source span ranges must be ordered') + } + if (span.sourceEndSu - span.sourceStartSu !== span.transform.rawLengthSu) { + throw new Error('PTY source span raw length does not match its source range') + } + if (!span.transform.transformed && span.data.length !== span.transform.rawLengthSu) { + throw new Error('Untransformed PTY source span length is invalid') + } + if ( + span.splittable !== undefined && + span.indivisible !== undefined && + span.splittable === span.indivisible + ) { + throw new Error('PTY source span split metadata is contradictory') + } +} + +export function assertPtySourceAck(ack: PtySourceCreditAck): void { + if (!ack.id || !ack.deliveryToken) { + throw new Error('PTY source ACK identity is incomplete') + } + assertPositiveSafeInteger(ack.clientGeneration, 'clientGeneration') + assertPositiveSafeInteger(ack.ownerGeneration, 'ownerGeneration') + assertNonNegativeSafeInteger(ack.creditedEndSu, 'creditedEndSu') +} diff --git a/src/shared/pty-source-receiving-activation.ts b/src/shared/pty-source-receiving-activation.ts new file mode 100644 index 00000000000..eb971cbc764 --- /dev/null +++ b/src/shared/pty-source-receiving-activation.ts @@ -0,0 +1,52 @@ +export type PtySourceReceivingActivation = Readonly<{ + status: 'pending' + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string + checkpointSourceEndSu: number + recoveryEndSu: number +}> + +export function parsePtySourceReceivingActivation( + value: unknown +): PtySourceReceivingActivation | undefined { + if (value === undefined) { + return undefined + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Invalid SSH PTY source activation response') + } + const input = value as Record + if ( + input.status !== 'pending' || + typeof input.deliveryToken !== 'string' || + input.deliveryToken.length === 0 || + typeof input.ptyIncarnation !== 'string' || + input.ptyIncarnation.length === 0 || + !positiveInteger(input.clientGeneration) || + !positiveInteger(input.ownerGeneration) || + !nonNegativeInteger(input.checkpointSourceEndSu) || + !nonNegativeInteger(input.recoveryEndSu) || + Number(input.recoveryEndSu) < Number(input.checkpointSourceEndSu) + ) { + throw new Error('Invalid SSH PTY source activation response') + } + return Object.freeze({ + status: 'pending', + clientGeneration: Number(input.clientGeneration), + ownerGeneration: Number(input.ownerGeneration), + ptyIncarnation: input.ptyIncarnation, + deliveryToken: input.deliveryToken, + checkpointSourceEndSu: Number(input.checkpointSourceEndSu), + recoveryEndSu: Number(input.recoveryEndSu) + }) +} + +function positiveInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) > 0 +} + +function nonNegativeInteger(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) >= 0 +} diff --git a/src/shared/pty-source-recovery-contract.ts b/src/shared/pty-source-recovery-contract.ts new file mode 100644 index 00000000000..2634072396c --- /dev/null +++ b/src/shared/pty-source-recovery-contract.ts @@ -0,0 +1,29 @@ +export type PtySourceRecoveryCheckpoint = Readonly<{ + status: 'checkpoint' + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string + acceptedSourceEndSu: number +}> + +export type PtySourceRecoveryRequest = + | PtySourceRecoveryCheckpoint + | Readonly<{ status: 'checkpointUnavailable' }> + +export type PtySourceRecoveryPending = Readonly<{ + status: 'pending' + clientGeneration: number + ownerGeneration: number + ptyIncarnation: string + deliveryToken: string + checkpointSourceEndSu: number + recoveryEndSu: number +}> + +export type PtySourceRecoveryResult = + | PtySourceRecoveryPending + | Readonly<{ status: 'restoreRequired'; reason: string }> + +export type PtySourceRecoveryComplete = Omit & + Readonly<{ id: string }> diff --git a/src/shared/relay-frame-buffer.ts b/src/shared/relay-frame-buffer.ts new file mode 100644 index 00000000000..5083804f1e1 --- /dev/null +++ b/src/shared/relay-frame-buffer.ts @@ -0,0 +1,84 @@ +export class RelayFrameBuffer { + private chunks: Buffer[] = [] + private bytes = 0 + + get length(): number { + return this.bytes + } + + append(chunk: Buffer): void { + this.chunks.push(chunk) + this.bytes += chunk.length + } + + clear(): void { + this.chunks = [] + this.bytes = 0 + } + + drain(): Buffer { + const out = this.chunks.length === 1 ? this.chunks[0] : Buffer.concat(this.chunks, this.bytes) + this.clear() + return out + } + + peek(count: number): Buffer { + const first = this.chunks[0] + if (first.length >= count) { + return first + } + const out = Buffer.allocUnsafe(count) + let copied = 0 + for (const part of this.chunks) { + copied += part.copy(out, copied, 0, Math.min(part.length, count - copied)) + if (copied >= count) { + break + } + } + return out + } + + take(count: number): Buffer { + const first = this.chunks[0] + if (first.length === count) { + this.chunks.shift() + this.bytes -= count + return first + } + if (first.length > count) { + this.chunks[0] = first.subarray(count) + this.bytes -= count + return first.subarray(0, count) + } + const out = Buffer.allocUnsafe(count) + let copied = 0 + while (copied < count) { + const part = this.chunks[0] + const take = Math.min(part.length, count - copied) + part.copy(out, copied, 0, take) + copied += take + if (take === part.length) { + this.chunks.shift() + } else { + this.chunks[0] = part.subarray(take) + } + } + this.bytes -= count + return out + } + + discard(count: number): void { + let remaining = count + while (remaining > 0) { + const part = this.chunks[0] + if (part.length <= remaining) { + this.chunks.shift() + remaining -= part.length + } else { + this.chunks[0] = part.subarray(remaining) + remaining = 0 + } + } + this.bytes -= count + } +} diff --git a/src/shared/relay-frame-decoder-contract.ts b/src/shared/relay-frame-decoder-contract.ts new file mode 100644 index 00000000000..6539c180e89 --- /dev/null +++ b/src/shared/relay-frame-decoder-contract.ts @@ -0,0 +1,52 @@ +export type DecodedFrame = { + type: number + id: number + ack: number + payload: Buffer +} + +export class FrameDecoderContinuationError extends Error { + readonly cause: unknown + + constructor(cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause) + super(`Frame decoder continuation failed: ${detail}`) + this.name = 'FrameDecoderContinuationError' + this.cause = cause + } +} + +export function publishFrameDecoderError( + observer: ((error: Error) => void) | null, + error: Error +): void { + try { + observer?.(error) + } catch { + // Error observers cannot escape decoder ownership. + } +} + +export function containFrameDecoderContinuation( + reset: () => void, + observer: ((error: Error) => void) | null, + cause: unknown +): void { + try { + reset() + } catch { + // Reset clears retained state before releasing paused read ownership. + } + publishFrameDecoderError(observer, new FrameDecoderContinuationError(cause)) +} + +export type FrameDecoderOptions = { + maxFramesPerTurn?: number + maxBytesPerTurn?: number + maxTurnMs?: number + now?: () => number + schedule?: (callback: () => void) => unknown + cancelScheduled?: (handle: unknown) => void + pause?: () => void + resume?: () => void +} diff --git a/src/shared/terminal-output-source-range.ts b/src/shared/terminal-output-source-range.ts new file mode 100644 index 00000000000..84a0f0b55a8 --- /dev/null +++ b/src/shared/terminal-output-source-range.ts @@ -0,0 +1,50 @@ +import type { PtySourceDeliveryIdentity, PtySourceSpan } from './pty-source-credit-contract' +import { + assertNonNegativeSafeInteger, + assertPtySourceIdentity +} from './pty-source-credit-validation' + +export type TerminalOutputSourceRange = Readonly< + Omit & { + splittable: boolean + } +> + +export function assertTerminalOutputSourceRange(range: TerminalOutputSourceRange): void { + assertPtySourceIdentity(range) + if (!range.spanId) { + throw new Error('Terminal output source range requires a span ID') + } + assertNonNegativeSafeInteger(range.sourceStartSu, 'sourceStartSu') + assertNonNegativeSafeInteger(range.sourceEndSu, 'sourceEndSu') + assertNonNegativeSafeInteger(range.displayStart, 'displayStart') + assertNonNegativeSafeInteger(range.displayEnd, 'displayEnd') + assertNonNegativeSafeInteger(range.transform.rawLengthSu, 'rawLengthSu') + if ( + range.sourceEndSu <= range.sourceStartSu || + range.displayEnd < range.displayStart || + range.sourceEndSu - range.sourceStartSu !== range.transform.rawLengthSu || + typeof range.splittable !== 'boolean' || + typeof range.transform.transformed !== 'boolean' || + typeof range.transform.scalarSafe !== 'boolean' || + (range.transform.transformed && range.splittable) || + (!range.transform.transformed && + range.sourceEndSu - range.sourceStartSu !== range.displayEnd - range.displayStart) + ) { + throw new Error('Terminal output source range is malformed') + } +} + +export function sameTerminalOutputSourceIdentity( + left: PtySourceDeliveryIdentity, + right: PtySourceDeliveryIdentity +): boolean { + return ( + left.id === right.id && + left.providerGeneration === right.providerGeneration && + left.clientGeneration === right.clientGeneration && + left.ownerGeneration === right.ownerGeneration && + left.ptyIncarnation === right.ptyIncarnation && + left.deliveryToken === right.deliveryToken + ) +} diff --git a/tests/e2e/helpers/remote-terminal-source-range-contract-fixture.ts b/tests/e2e/helpers/remote-terminal-source-range-contract-fixture.ts new file mode 100644 index 00000000000..dbb2647d67e --- /dev/null +++ b/tests/e2e/helpers/remote-terminal-source-range-contract-fixture.ts @@ -0,0 +1,60 @@ +import type { TerminalOutputSourceRange } from '../../../src/shared/terminal-output-source-range' +import { TerminalSourceRangeLedger } from '../../../src/main/runtime/rpc/terminal-source-range-ledger' + +export type RemoteTerminalContractTopology = 'headed-desktop-server' | 'headless-serve' + +export function createRemoteTerminalSourceRangeContractFixture( + topology: RemoteTerminalContractTopology +) { + const settled: TerminalOutputSourceRange[] = [] + const transferred: TerminalOutputSourceRange[] = [] + let generationCounter = 0 + let active: + | { + generation: string + ledger: TerminalSourceRangeLedger + } + | undefined + + return { + topology, + evidence: 'deterministic-contract-fixture' as const, + hostPtyIdentity: 'host-owned-pty', + connect() { + const generation = `${topology}:stream:${++generationCounter}` + active = { generation, ledger: new TerminalSourceRangeLedger(generation) } + return generation + }, + accept(encodedBytes: number, ranges: readonly TerminalOutputSourceRange[]) { + const displayLength = + ranges.length > 0 ? ranges.at(-1)!.displayEnd - ranges[0]!.displayStart : 0 + return active?.ledger.accept(encodedBytes, displayLength, ranges) ?? null + }, + acknowledge(generation: string, ackedEndByte: number) { + if (!active) { + throw new Error('remote_terminal_fixture_disconnected') + } + const result = active.ledger.acknowledge(generation, ackedEndByte) + if (result.status === 'accepted') { + settled.push(...result.settled) + } + return result + }, + detach() { + if (!active) { + return + } + const transfer = active.ledger.beginTransfer() + transferred.push(...transfer.frames.flatMap((frame) => frame.sourceRanges)) + transfer.commit() + active = undefined + }, + snapshot() { + return { + settled: settled.slice(), + transferred: transferred.slice(), + active: active?.ledger.getDebugSnapshot() ?? null + } + } + } +} diff --git a/tests/e2e/helpers/remote-terminal-source-range-contract-fixture.unit.test.ts b/tests/e2e/helpers/remote-terminal-source-range-contract-fixture.unit.test.ts new file mode 100644 index 00000000000..dfe753dda2c --- /dev/null +++ b/tests/e2e/helpers/remote-terminal-source-range-contract-fixture.unit.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import type { TerminalOutputSourceRange } from '../../../src/shared/terminal-output-source-range' +import { + createRemoteTerminalSourceRangeContractFixture, + type RemoteTerminalContractTopology +} from './remote-terminal-source-range-contract-fixture' + +function sourceRange(): TerminalOutputSourceRange { + return { + id: 'pty-1', + spanId: 'span-1', + providerGeneration: 8, + clientGeneration: 5, + ownerGeneration: 3, + ptyIncarnation: 'incarnation-1', + deliveryToken: 'token-1', + sourceStartSu: 0, + sourceEndSu: 4, + displayStart: 0, + displayEnd: 4, + splittable: true, + transform: { transformed: false, rawLengthSu: 4, scalarSafe: true } + } +} + +describe.each(['headed-desktop-server', 'headless-serve'])( + '%s remote terminal source-range contract', + (topology) => { + it('keeps host identity and rejects a stale client generation after reconnect', () => { + const fixture = createRemoteTerminalSourceRangeContractFixture(topology) + const oldGeneration = fixture.connect() + fixture.accept(4, [sourceRange()]) + fixture.detach() + const generation = fixture.connect() + fixture.accept(4, [sourceRange()]) + + expect(fixture.hostPtyIdentity).toBe('host-owned-pty') + expect(fixture.evidence).toBe('deterministic-contract-fixture') + expect(fixture.acknowledge(oldGeneration, 4).status).toBe('stale-generation') + expect(fixture.acknowledge(generation, 4).status).toBe('accepted') + expect(fixture.snapshot()).toMatchObject({ + settled: [{ spanId: 'span-1' }], + transferred: [{ spanId: 'span-1' }] + }) + }) + } +) diff --git a/tests/e2e/ssh-docker-relay-perf.spec.ts b/tests/e2e/ssh-docker-relay-perf.spec.ts index 1d7606fcbe3..775f7cc5f62 100644 --- a/tests/e2e/ssh-docker-relay-perf.spec.ts +++ b/tests/e2e/ssh-docker-relay-perf.spec.ts @@ -217,12 +217,14 @@ test.describe('Docker SSH relay perf', () => { `node -e ${shellQuote(remoteTypingLoadScript(activeRunId))}` ) await waitForTerminalOutput(orcaPage, `REMOTE_TUI_READY_${activeRunId}`, 30_000, 80_000) - await expect - .poll(async () => (await readSshPtyAckGate(orcaPage))?.heldAckChars ?? 0, { + const heldAckPressure = expect.poll( + async () => (await readSshPtyAckGate(orcaPage))?.heldAckChars ?? 0, + { timeout: 30_000, message: 'remote background SSH PTY stream did not build held ACK pressure' - }) - .toBeGreaterThan(MIN_HELD_SSH_ACK_CHARS) + } + ) + await heldAckPressure.toBe(MIN_HELD_SSH_ACK_CHARS) const measurement = await measureRemoteTyping(orcaPage, activePtyId, activeRunId) const ackGate = await readSshPtyAckGate(orcaPage) @@ -238,7 +240,7 @@ test.describe('Docker SSH relay perf', () => { type: 'docker-ssh-relay-pty-ack-pressure', description: summary }) - expect(ackGate?.heldAckChars ?? 0).toBeGreaterThan(MIN_HELD_SSH_ACK_CHARS) + expect(ackGate?.heldAckChars ?? 0).toBe(MIN_HELD_SSH_ACK_CHARS) expect(measurement.medianLatencyMs).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS) expect(measurement.worstLatencyMs).toBeLessThan(MAX_WORST_KEY_LATENCY_MS) @@ -368,11 +370,26 @@ test.describe('Docker SSH relay perf', () => { const beforeMarker = `SSH_RECONNECT_BEFORE_${Date.now()}` await execInTerminal(orcaPage, beforePtyId, `printf ${shellQuote(beforeMarker)}`) await waitForTerminalOutput(orcaPage, beforeMarker, 20_000, 60_000) + const recoveryStartedMarker = `SSH_RECONNECT_RECOVERY_STARTED_${Date.now()}` + const recoveryMarker = `SSH_RECONNECT_RECOVERY_${Date.now()}` + const recoveryScript = [ + 'let frame = 0', + "const chunk = 'Q'.repeat(4096)", + `process.stdout.write('${recoveryStartedMarker}\\n')`, + 'const timer = setInterval(() => {', + 'frame += 1', + "process.stdout.write('RECOVERY_FRAME_' + frame + '_' + chunk + '\\n')", + `if (frame === 256) { clearInterval(timer); process.stdout.write('${recoveryMarker}\\n') }`, + '}, 10)' + ].join(';') + await execInTerminal(orcaPage, beforePtyId, `node -e ${shellQuote(recoveryScript)}`) + await waitForTerminalOutput(orcaPage, recoveryStartedMarker, 30_000, 80_000) await reconnectDockerSshRelayTarget(orcaPage, remote.targetId) await ensureTerminalVisible(orcaPage, 45_000) await waitForActiveTerminalManager(orcaPage, 60_000) const afterPtyId = await waitForActivePanePtyId(orcaPage, 60_000) + await waitForTerminalOutput(orcaPage, recoveryMarker, 30_000, 80_000) const afterMarker = `SSH_RECONNECT_AFTER_${Date.now()}` const remoteProofPath = `/tmp/${afterMarker}` await execInTerminal(