starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-10-19 16:03:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ssh): close recovery ownership gaps

Browse Source
This commit is contained in:
OrcaWin
2026-07-28 20:13:37 -07:00
parent ddc60bf62b
commit 5611bb45b5
28 changed files with 1507 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/reliability-gates.jsonc
+85 -81
View File
@@ -217,26 +217,12 @@
"direct SSH host catalog authority",
"direct SSH generation-scope rollover"
],
"platforms": [
"macos",
"linux",
"windows"
],
"providers": [
"local",
"ssh"
],
"coveredPlatforms": [
"macos"
],
"coveredProviders": [
"local",
"ssh"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["local", "ssh"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["local", "ssh"],
"coverageNotes": "Local deterministic evidence covers git-common classification, desktop watcher debounce counts, non-overlapping poller semantics, macOS native-watch fallback, preload cleanup, Source Control active-visible repo filtering, the direct SSH five-slot fair scheduler, timeout barrier, aggregate privacy schema, coordinator-to-renderer telemetry wiring, host-catalog provenance rejection, and process generation-scope rollover across sibling targets. A macOS Electron client completed a direct SSH disconnect/reconnect against an ephemeral Linux Docker target with exact host/authority hydration and remote proof-file verification. Linux/Windows desktop clients, multi-target live fanout, paired-client, and WSL runs remain gaps.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/7086"
],
"motivatingLinks": ["https://github.com/stablyai/orca/pull/7086"],
"invariant": "Index-only Git activity below the common Git directory must not emit worktrees:changed, invalidate worktree caches, or trigger fetchWorktrees fanout; structural add/remove/HEAD/gitdir/locked/config.worktree changes must still refresh worktrees and nudge Source Control; external head moves (commit, amend, reset) must reach background worktree rows through spawn-free metadata reads, never through structural fanout. Direct SSH reconnect discovery must stay host- and authority-qualified, reject contradictory main-catalog provenance without returning rows, admit at most five locally unsettled provider calls, retain a retrying timeout barrier, and emit one identifier-free aggregate product event per target operation. A process generation-scope rollover revokes every direct SSH target and old-scope provider request, not only the target whose counter exhausted.",
"oracle": "Classify exact git-common paths as structural, status-only, or ignored; count notifications from debounced watcher events; force the Linux/Windows poll path to emit allowlisted leaf events, detect linked HEAD rewrites independent of entry-directory mtime, and surface in-place index rewrites via the backstop re-stat; diff head identities from metadata-file reads and notify only real head moves; assert Source Control subscribes to both structural and status-only signals with active-repo and visibility filters. For direct SSH, reject catalog rows whose explicit and legacy host provenance contradict, roll one exhausted target into a fresh process generation scope while invalidating sibling target tokens, count locally unsettled attempts and round-robin admissions, keep lineage blocked through the first timeout retry, distinguish timeout/rejection/cancel/stale results, and reject telemetry properties carrying target, repo, host, path, label, user, request, lease, terminal, or raw-error data.",
"commands": [
@@ -3103,14 +3089,14 @@
"https://github.com/stablyai/orca/pull/7002",
"https://github.com/stablyai/orca/pull/7054"
],
"invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; provisional activation data cannot project before authority commits, pending renderer projection IDs are capped and compacted across split remainders, exit preparation owns its renderer fence through finalization, exit-time cancellation transfers published projections before proof commit, generation close fences late proof while draining exact projection waiters, and pause/resume targets only the exact provider generation. Additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.",
"oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, a 1,024-ID projection cap with split-remainder compaction, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, exact provider-generation pause ownership, renderer exit/data ordering, and ACK-gated in-flight bounds. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations without releasing in-flight byte credit. Replacement reservation rolls back every earlier span on a later failure, commit requires the exact frozen transfer states, and authoritatively reclaimed covered spans complete idempotently. The exit-deadline oracle publishes a source-backed desktop projection, advances a fake clock to cancellation, and requires one projection transfer, proof commit, preparation, and final exit with no provider close or retained obligation; a second controlled proof remains pending across generation close and must never publish final exit, while both paths release the renderer preparation lease exactly once. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, claim-gated projection under same-decoder-turn response/data delivery, exact early-ACK reservation through send settlement or same-token retry, recovery-completion settlement before live admission, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved gate-off primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.",
"invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; provisional activation data cannot project, reconnect may release it only into the exact attempt's private recovery quarantine after contract validation, and exit seals that quarantine against later same-token frames until ordered admission or exact cancellation proof. Pending renderer projection IDs are capped and compacted across split remainders, exit preparation owns its renderer fence through finalization, exit-time cancellation transfers published projections before proof commit, generation close fences late proof while draining exact projection waiters, closed-generation identity compacts without weakening stale rejection, canceled source-delivery retirement retains at most one ordered token per PTY, and pause/resume targets only the exact provider generation. Additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.",
"oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, a 1,024-ID projection cap with split-remainder compaction, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, exact provider-generation pause ownership, renderer exit/data ordering, and ACK-gated in-flight bounds. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations without releasing in-flight byte credit. Replacement reservation rolls back every earlier span on a later failure, commit requires the exact frozen transfer states, and authoritatively reclaimed covered spans complete idempotently. The exit-deadline oracle publishes a source-backed desktop projection, advances a fake clock to cancellation, and requires one projection transfer, proof commit, preparation, and final exit with no provider close, retained obligation, duplicate finalization, or process-lifetime cancellation tombstone; a second controlled proof remains pending across generation close and must never publish final exit, while both paths release the renderer preparation lease exactly once. The intake additionally closes 2,048 sequential provider generations into one exact range, rejects stale events across that range, and preserves an out-of-order live gap until its own close. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, claim-gated projection under same-decoder-turn response/data delivery, exact early-ACK reservation through send settlement or same-token retry, recovery completion visible before lease-held frames transfers those frames only into the private quarantine and admits no output until exact checkpoint-to-recoveryEnd coverage, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved gate-off primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.",
"commands": [
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-pending-projection-admissions.test.ts src/main/ipc/ssh-pty-legacy-projection.test.ts src/main/ipc/ssh-pty-model-admission.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime-path-candidate-history.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/orca-runtime-tail-wait-memo.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/relay/git-response-stream-ownership.test.ts src/relay/pty-handler.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-recovery-interleavings.test.ts src/relay/relay-pty-source-recovery-completion.test.ts src/relay/relay-pty-source-restore-retry.test.ts --reporter=dot",
@@ -3119,9 +3105,11 @@
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-legacy-projection.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-generation-guard.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot",
"pnpm exec vitest run src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts --reporter=dot",
"ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
],
"testFiles": [
@@ -3133,6 +3121,7 @@
"src/main/runtime/orca-runtime-tail-wait-memo.test.ts",
"src/main/providers/ssh-pty-provider.test.ts",
"src/main/providers/ssh-pty-notification-routing.test.ts",
"src/main/providers/ssh-pty-source-delivery-ledger.test.ts",
"src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts",
"src/main/providers/ssh-pty-provider-exit-race.test.ts",
"src/main/ssh/ssh-relay-session.test.ts",
@@ -3142,6 +3131,7 @@
"src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts",
"src/main/ssh/ssh-relay-session-terminal-error.test.ts",
"src/main/ssh/ssh-pty-recovery-retention-budget.test.ts",
"src/main/ssh/ssh-pty-retired-source-deliveries.test.ts",
"src/main/ssh/relay-protocol-backpressure.test.ts",
"src/relay/protocol-backpressure.test.ts",
"src/relay/git-response-stream-ownership.test.ts",
@@ -3158,6 +3148,7 @@
"src/shared/pty-consumer-session.test.ts",
"src/main/ipc/ssh-pty-model-admission.test.ts",
"src/main/ipc/ssh-pty-output-intake.test.ts",
"src/main/ipc/ssh-pty-output-generation-guard.test.ts",
"src/main/ipc/ssh-pty-output-exit-deadline.test.ts",
"src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts",
"src/main/runtime/rpc/terminal-source-range-ledger.test.ts",
@@ -3223,9 +3214,19 @@
"file": "src/main/providers/ssh-pty-notification-routing.test.ts",
"assertions": [
"provisional source frames remain unprojected and are discarded on rollback",
"held and later recovery frames route only to the private sink until final commit, without ordinary listeners or live PTY publication",
"exit during private recovery retires the activation ledger at final commit and rejects late same-token frames",
"private recovery retirement restores the exact predecessor without issuing a second cancellation",
"older rollback cancels only its exact token without replacing a newer activation"
]
},
{
"file": "src/main/providers/ssh-pty-source-delivery-ledger.test.ts",
"assertions": [
"exit seals a provisional or private-recovery lease against later same-token admission",
"a stale transfer retains rollback authority and requests exact cancellation instead of orphaning its token"
]
},
{
"file": "src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts",
"assertions": [
@@ -3347,7 +3348,15 @@
"exit remains behind accepted model, desktop projection, and required remote obligations",
"exit timeout cancels only the matching delivery and keeps unrelated provider work usable",
"desktop source identity and scanner facts commit atomically or roll back without credit",
"renderer exit preparation remains owned through finalization and releases on duplicate, failure, or generation close"
"renderer exit preparation remains owned through finalization and releases on duplicate, failure, or generation close",
"2,048 monotonic provider closes compact into one exact stale-generation range while out-of-order live gaps remain admissible"
]
},
{
"file": "src/main/ipc/ssh-pty-output-generation-guard.test.ts",
"assertions": [
"sequential generation closure compacts to one exact range without weakening stale rejection",
"out-of-order closures expose the exact unclosed generation count below the closed high-water"
]
},
{
@@ -3395,7 +3404,7 @@
"file": "src/main/ssh/ssh-relay-session-data-delivery.test.ts",
"assertions": [
"the state-machine stale-owner error clears cached state and retries exactly once without resume",
"recovery remains quarantined until exact checkpoint-to-recoveryEnd coverage and live handoff",
"completion visible before lease-held recovery frames transfers those frames only into private quarantine and projects them once after exact checkpoint-to-recoveryEnd coverage and live handoff",
"invalid recovery cancels only its replacement token without physical PTY or ownership teardown"
]
},
@@ -3404,10 +3413,18 @@
"assertions": [
"empty recovery retains recoveryEndSu as the first live-frame continuity anchor",
"validated token cancellation drops queued late frames without physical PTY teardown",
"rejected recovery cancellation publication or proof closes only the exact provider generation and releases its provider, publishers, mux, and activation lease once",
"after recovery quarantine ownership transfer, rejected cancellation publication or proof closes only the exact provider generation and releases its provider, publishers, mux, and activation state once",
"negative, credited-ahead, checkpoint-mismatched, and under-covering recovery cancellation proofs fail closed against the highest privately observed range",
"overlapping recovery never cancels on or mutates the replacement mux, checkpoint, lease, provider state, or ownership"
]
},
{
"file": "src/main/ssh/ssh-pty-retired-source-deliveries.test.ts",
"assertions": [
"10,000 ordered token cancellations for one PTY retain one latest-token retirement record",
"the next activation or PTY exit boundary clears only that PTY's retirement record"
]
},
{
"file": "src/main/ssh/ssh-multiplexer-transport-writer.test.ts",
"assertions": [
@@ -3483,13 +3500,13 @@
"summary": "2 test files passed, 609 tests passed on pushed commit 4fb14eac3897."
},
{
"date": "2026-07-28",
"date": "2026-07-29",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-terminal-error.test.ts src/main/ssh/ssh-pty-recovery-retention-budget.test.ts src/main/ssh/relay-protocol-backpressure.test.ts src/relay/protocol-backpressure.test.ts src/relay/pty-source-credit-ledger.test.ts src/relay/pty-source-credit-scheduler.test.ts src/relay/relay-pty-source-publication.test.ts src/relay/ssh-pty-source-credit-adapter.test.ts --reporter=dot",
"result": "passed",
"durationSeconds": 6.02,
"summary": "Thirteen deterministic SSH relay/session/source/decoder files passed 176 tests, including state-machine-driven stale-owner fallback, scheduler rejection isolation, and typed decoder-continuation containment; no live topology was exercised."
"durationSeconds": 2.24,
"summary": "Sixteen deterministic SSH relay/session/source/decoder files passed 192 tests, including exit-sealed private recovery, retained stale-transfer cancellation authority, exact private-frame proof watermarks, one retirement record across 10,000 same-PTY token rotations, stale-owner fallback, and scheduler rejection isolation; no live topology was exercised."
},
{
"date": "2026-07-28",
@@ -3510,22 +3527,40 @@
"summary": "Three deterministic main/runtime files passed 78 tests, including joined renderer/headless snapshot admission, exact partial cumulative credit, and recovered-token continuity; no live paired server or Docker topology was exercised."
},
{
"date": "2026-07-28",
"date": "2026-07-29",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-generation-guard.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
"result": "passed",
"durationSeconds": 11.71,
"summary": "Eleven deterministic source-intake, exit-deadline, remote-consumer, multiplex, writer, rollout, and deploy files passed 197 tests, including exit-time ordering, control-lane drain priority, and typed mismatched-principal recovery with retained authority isolation."
"durationSeconds": 6.59,
"summary": "Twelve deterministic source-intake, generation, exit-deadline, remote-consumer, multiplex, writer, rollout, and deploy files passed 205 tests, including exact active-generation gaps, exit-time ordering, control-lane drain priority, and typed mismatched-principal recovery with retained authority isolation."
},
{
"date": "2026-07-29",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-source-delivery-ledger.test.ts src/main/ssh/ssh-pty-retired-source-deliveries.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot",
"result": "passed",
"durationSeconds": 6.89,
"summary": "Thirteen current-head deterministic files passed 150 tests across exit-sealed private recovery, stale-transfer cancellation authority, bounded latest-token retirement, restore retirement, fail-closed recovery cancellation, partial ACK and token rotation, reclaimed-span replacement, mux lane fairness, provisional activation, and exit-proof ordering; no live topology was exercised."
},
{
"date": "2026-07-28",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot",
"command": "pnpm exec vitest run src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ssh/ssh-relay-session-data-delivery.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts src/main/ssh/ssh-relay-session-incarnation.test.ts src/main/ssh/ssh-relay-session.test.ts --reporter=dot",
"result": "passed",
"durationSeconds": 12.5,
"summary": "Eleven current-head deterministic files passed 134 tests across restore retirement, fail-closed recovery cancellation, partial ACK and token rotation, reclaimed-span replacement, mux lane fairness, provisional activation, and exit-proof ordering; no live topology was exercised."
"durationSeconds": 1.86,
"summary": "Seven current-worktree provider/session files passed 94 tests after a red deterministic seam reproduced completion visibility before lease-held recovery data; the candidate transfers held and later frames only into private recovery quarantine, final-commits after fence/admission, retires exited activation state, and preserves fail-closed malformed, gapped, overlapping, missing-body, cancellation-proof, and replacement behavior."
},
{
"date": "2026-07-28",
"runner": "local",
"platform": "macos",
"command": "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"result": "passed",
"durationSeconds": 66,
"summary": "All four deployed Linux relay/direct-SSH cases passed against one artifact: direct typing 6.2/7.4 ms median/worst, ACK-stalled typing 5.5/16.3 ms at exactly 262,144 held source units, fixed-size filesystem/Git churn 141.2/151.5 ms with 95 bulk reads, and owner-lease reconnect in 16.0 seconds."
},
{
"date": "2026-07-28",
@@ -3543,15 +3578,15 @@
},
"flakeHistory": {
"status": "unknown",
"evidence": "Focused main-process backlog and direct-SSH reconnect/memory/decoder tests are deterministic unit slices. On 2026-07-27 the source-credit reconnect case passed once in the full four-test Docker suite (15.7 seconds) and three consecutive isolated repeat-each runs (21.1, 19.8, and 15.3 seconds); the combined perf soak still needs longer runtime history before promotion."
"evidence": "Focused main-process backlog and direct-SSH reconnect/memory/decoder tests are deterministic unit slices. On 2026-07-27 the source-credit reconnect case passed once in the full four-test Docker suite (15.7 seconds) and three consecutive isolated repeat-each runs (21.1, 19.8, and 15.3 seconds). On 2026-07-28 it passed against private recovery transfer in 17.7 seconds alone and 16.0 seconds in the full four-case suite; the combined perf soak still needs longer runtime history before promotion."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output plus projection IDs stay within exact caps, direct-SSH source/recovery memory and both frame decoders stay within exact caps, and 50-PTY reconnect isolates slow/failing siblings. On pre-fix 4bf54da9b, reconnect pause targeted the replacement provider, disposal omitted a pause-only generation, normal and generation-close exit paths never released the renderer preparation lease, and the projection-cap seam did not exist; the same oracles pass with exact generation indexing, paused-generation enumeration, barrier-owned exit leases, and the 1,024-ID transfer latch. The deterministic main-to-relay drain oracle failed with ordinary-2 and ordinary-3 ahead of a later control frame under the single FIFO, then passed with control-first lane selection and four-write ordinary fairness. On exact pre-fix baseline db167ea3d with test-only oracles, cancellation proof reclaimed the published projection span before transfer and generation close allowed a late proof to publish final exit; both deterministic oracles pass after proof acquisition and commit are separated under the exact deadline barrier. Before snapshot admission, the trailing span entered transferring instead of remaining open; after retaining model-sequence ends and reserving at the serialized fence, the joined main/runtime oracle passed for renderer and headless sources with upstream boundaries 4 then 8. Before cumulative partial ACK and source rotation, the ledger rejected ACK 40 of 100 and the multiplex seam detached instead of publishing recovered token B; both exact oracles pass while stale stream and source generations remain rejected. Before restore-record retirement, invalid-checkpoint recovery followed by restore/retry returned restoreRequired a second time instead of opening a fresh activation; the same deterministic oracle passes after exact response-settlement cleanup. With test-only oracles on pre-fix 4bf54da9b, an existing-owner mismatch lacked the stale-owner code and both shared authority and main fallback rows failed; the candidate types that refusal while stale fresh admission remains a subscriber and the retained principal and lease can still recover. Other controls previously failed with serial reattach, UTF-8-only accounting, and roughly 32 MiB decoder retention. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion."
"evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output plus projection IDs stay within exact caps, direct-SSH source/recovery memory and both frame decoders stay within exact caps, and 50-PTY reconnect isolates slow/failing siblings. On pre-fix 4bf54da9b, reconnect pause targeted the replacement provider, disposal omitted a pause-only generation, normal and generation-close exit paths never released the renderer preparation lease, and the projection-cap seam did not exist; the same oracles pass with exact generation indexing, paused-generation enumeration, barrier-owned exit leases, and the 1,024-ID transfer latch. The deterministic main-to-relay drain oracle failed with ordinary-2 and ordinary-3 ahead of a later control frame under the single FIFO, then passed with control-first lane selection and four-write ordinary fairness. On exact pre-fix baseline db167ea3d with test-only oracles, cancellation proof reclaimed the published projection span before transfer and generation close allowed a late proof to publish final exit; both deterministic oracles pass after proof acquisition and commit are separated under the exact deadline barrier. Before snapshot admission, the trailing span entered transferring instead of remaining open; after retaining model-sequence ends and reserving at the serialized fence, the joined main/runtime oracle passed for renderer and headless sources with upstream boundaries 4 then 8. Before cumulative partial ACK and source rotation, the ledger rejected ACK 40 of 100 and the multiplex seam detached instead of publishing recovered token B; both exact oracles pass while stale stream and source generations remain rejected. Before restore-record retirement, invalid-checkpoint recovery followed by restore/retry returned restoreRequired a second time instead of opening a fresh activation; the same deterministic oracle passes after exact response-settlement cleanup. With test-only oracles on pre-fix 4bf54da9b, an existing-owner mismatch lacked the stale-owner code and both shared authority and main fallback rows failed; the candidate types that refusal while stale fresh admission remains a subscriber and the retained principal and lease can still recover. Before the recovery-quarantine activation fix, completion was visible at source unit 4 while lease-held recovery ended at 8, so the fence rejected and cancellation failed; the same byte-identical seam now transfers held ranges only into private quarantine, admits output after exact body/fence validation, and the deployed Linux relay reconnect passes. Other controls previously failed with serial reattach, UTF-8-only accounting, and roughly 32 MiB decoder retention. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion."
},
"performanceBudget": {
"required": true,
"evidence": "Suggested ceilings: renderer in-flight <=8MB total, <=512KB per PTY plus active reserve, renderer queued chars <=2MB, dropped backlogs 0, hidden restore <=1000ms, active key median/worst <=75ms/300ms in perf scenarios. Restore-record retirement adds one exact map-identity check at response settlement with no polling, timers, scans, subprocesses, or provider calls. The Docker SSH gate enforces median/worst typing below 500/2,000 ms and observed 3.6/107.3 ms during an exact 256 KiB stalled-credit plateau plus 148.1/161.1 ms during 93 completed filesystem/Git bulk reads. With 1,024 retained provenance candidates, 4,096 pathless chunks dropped from 78.84ms and 4,096 array replacements to 1.74ms and zero replacements. Brace-free 1 KiB output dropped from 20.32/79.12/318.23ms to 0.31/0.64/2.35ms across 4,096/16,384/65,536 chunks. Repeated ordinary 252,000-character tail checks dropped from 96.34/464.20/941.79/3,796.70ms to 3.21/15.68/31.34/129.72ms across 100/500/1,000/4,096 updates."
"evidence": "Suggested ceilings: renderer in-flight <=8MB total, <=512KB per PTY plus active reserve, renderer queued chars <=2MB, dropped backlogs 0, hidden restore <=1000ms, active key median/worst <=75ms/300ms in perf scenarios. Restore-record retirement adds one exact map-identity check at response settlement with no polling, timers, scans, subprocesses, or provider calls. The Docker SSH gate enforces median/worst typing below 500/2,000 ms; the current private-transfer artifact observed 5.5/16.3 ms during an exact 256 KiB stalled-credit plateau, 141.2/151.5 ms during 95 completed filesystem/Git bulk reads, and a 16.0-second reconnect. With 1,024 retained provenance candidates, 4,096 pathless chunks dropped from 78.84ms and 4,096 array replacements to 1.74ms and zero replacements. Brace-free 1 KiB output dropped from 20.32/79.12/318.23ms to 0.31/0.64/2.35ms across 4,096/16,384/65,536 chunks. Repeated ordinary 252,000-character tail checks dropped from 96.34/464.20/941.79/3,796.70ms to 3.21/15.68/31.34/129.72ms across 100/500/1,000/4,096 updates."
},
"promotionCriteria": [
"Split daemon stream backpressure into a deterministic provider/IPC contract if full E2E is flaky.",
@@ -3745,23 +3780,10 @@
"provider reattach incarnation fencing",
"unknown liveness"
],
"platforms": [
"macos",
"linux",
"windows"
],
"providers": [
"ssh",
"remote-runtime",
"wsl"
],
"coveredPlatforms": [
"macos"
],
"coveredProviders": [
"ssh",
"remote-runtime"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["ssh", "remote-runtime", "wsl"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["ssh", "remote-runtime"],
"coverageNotes": "Deterministic renderer coverage proves startup publishes the state returned by ssh.connect, retained native and runtime SSH payloads are admitted through production routes only with valid complete authority, stale cleanup cannot unregister a replacement runtime terminal, direct SSH Git and folder panes clear and retry by exact authority, one authority chain stops after two automatic attempts even when each timeout exceeds the rolling window, rejected acknowledgements mutate no store maps, and one shared exact attempt admits every concurrent split-pane spawn and reattach while preserving the first PTY as the tab fallback. A later sibling failure rotates the tab once, stale callbacks from the prior attempt mutate no state, split remount activity suppression is counted per leaf, primary PTY exit promotes a bound survivor or preserves an empty continuation gap for a late sibling, and primary, non-primary, or null-PTY detach preserves exact authority on both resulting tabs. Intentional pane disposal cancels its settlement timer without breaking StrictMode remount timeout ownership. Target snapshot hydration/reconnect preserves sibling SSH/local/WSL/runtime state, and a mounted remote-runtime terminal survives repeated transport partitions without changing PTY identity. Direct SSH coordinator tests cover immediate terminal finalization, hydration correction, damping, bounded retry, and telemetry non-interference. Client/server heartbeat tests cover timer suspension, socket generations fence stale callbacks, cold restored-terminal attachment retries, cached pixels remain unhealthy until authoritative replay, automatic retries stop after one minute, manual reconnect preserves the PTY, and pane closure releases recovery UI state. Current macOS Electron journeys against an ephemeral Linux Docker SSH target cover exact-authority repo/worktree hydration, live terminal recovery after disconnect/reconnect, and eager six-terminal remount after renderer reload. A Windows remote-runtime smoke covers reachability and PTY round-trip. Multi-target live fanout, paired-close, WSL, and patched live partition journeys remain gaps.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/6951",
@@ -5137,30 +5159,12 @@
"terminal tab materialization",
"workspace re-entry"
],
"platforms": [
"macos",
"linux",
"windows"
],
"providers": [
"local",
"daemon",
"ssh",
"wsl",
"remote-runtime"
],
"coveredPlatforms": [
"macos"
],
"coveredProviders": [
"local",
"daemon",
"ssh"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["local", "daemon", "ssh"],
"coverageNotes": "Deterministic units cover local worker presentation, reveal-failure warnings, stable-pane Run/Dispatch routing, and the SSH in-process CLI fallback. An isolated macOS Electron journey launches a fake Codex worker through the real RPC path, tolerates spawn-time handle reminting, asserts the inactive tab in the DOM before navigation, checks Run delivery by pane identity, and proves workspace re-entry keeps one worker tab by both original tab ID and visible title. SSH, WSL, remote-runtime, Linux, and Windows remain live-test gaps; federated workers retain explicit background presentation.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/11107#discussion_r3663321387"
],
"motivatingLinks": ["https://github.com/stablyai/orca/pull/11107#discussion_r3663321387"],
"invariant": "Starting a worker in the coordinator's current workspace must materialize one inactive terminal tab before worker-start returns, preserve coordinator focus, and remain exactly once after workspace re-entry. An exact existing target workspace must receive a discoverable tab without stealing coordinator focus; if renderer reveal fails, worker-start must expose that the live worker remains background-only. Run and Dispatch checks must resolve through the caller's stable pane identity when a terminal handle is reminted, while a live handle outranks mismatched pane metadata, explicit legacy terminal inspection remains handle-scoped, and remote or headless worker presentation remains background-only.",
"oracle": "Drive Run create, Task create, and worker-start through a production Electron runtime with a deterministic Codex fixture. Require the worker tab to be visible in the DOM with data-active=false while the coordinator tab stays active, send ACK to the Run, read it with a deliberately stale coordinator handle plus its stable pane key, switch workspaces away and back, and require exactly one worker tab by original ID and visible title. Unit tests separately assert local worker-start omits background presentation, reveal failures return an actionable warning without discarding the worker, reminted coordinators and workers retain mailbox routing, live handles outrank mismatched pane metadata, and explicit legacy terminal checks do not inherit the caller pane key locally or through the SSH fallback.",
"commands": [
docs/plans/2026-07-27-ssh-relay-pty-backpressure.md
+192 -76
View File
@@ -8,7 +8,7 @@ HEAD; live topology gaps remain explicit below
Historical unbounded baseline: `badf91101babf96fa09cb79a8294f7e23b9f081c`
(the implementation branch parent). The implementation was rebased onto
`origin/main@681c4ba458d28f2f9b527db1cb6b246baf7bf478`; final GitHub CI remains
`origin/main@c8dba6d72caa60aeb663ec0cd525d0216e485f63`; final GitHub CI remains
the merge gate.
## Scope
@@ -47,8 +47,9 @@ implemented. In this PR:
- Spawn and attach responses carry immutable `sourceActivation` metadata. The
main mux installs a provisional receive lease synchronously in
`beforeResolve`; adjacent source notifications remain private to that lease
until provider validation commits, while rollback drops them and waits for
exact token-cancellation proof.
until spawn validation commits or a validated reconnect contract transfers
them into its private recovery quarantine. Rollback drops untransferred data
and waits for exact token-cancellation proof.
- `SshRelaySession` owns grant/reconnect state, bounded recovery quarantine,
exact recovery fencing, and cumulative ACK publication through the main SSH
mux. Failed recovery-cancellation publication or proof closes the exact
@@ -73,7 +74,7 @@ The implementation is intentionally scoped:
| Surface | State in this PR | Evidence |
| ------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------ |
| Same-build direct SSH/deployed relay, V1 on | implemented | current-head deterministic relay/main contracts plus prior macOS-hosted Docker OpenSSH |
| Same-build direct SSH/deployed relay, V1 on | implemented | current-head deterministic relay/main contracts; exact-current-artifact Docker rerun pending |
| Same-build SSH, V1 off | implemented bounded legacy mode | deterministic rollout and writer contracts |
| Reconnect/owner recovery | implemented | exact contiguous recovery, fail-closed cancellation, eight-wide reattach tests, Docker reconnect |
| Headed/headless remote consumers | source-range rotation/replacement implemented | deterministic runtime/main seam only; no live paired-runtime claim |
@@ -88,9 +89,10 @@ not prove headed or headless paired-runtime behavior, WSL, Windows ConPTY or
named pipes, local daemon/provider behavior, folder workspaces, prior-version
processes, mixed-version clients, or the Ubuntu 20.04 packaging floor.
The prior Docker evidence remains useful but predates the final lifecycle
fixes. Current-head deterministic checks and document validation are listed
below; no new live topology proof is inferred from them.
The recorded Docker rerun used the private-recovery-transfer checkpoint. It
predates the final exit-sealed recovery, exact private-watermark, and bounded
retirement fixes, so it is not exact-current-artifact evidence. No additional
live topology proof is inferred from that run.
## Verified baseline and migration boundary
@@ -211,8 +213,10 @@ renderer-only cap.
5. A spawn/attach token remains `activating` until its metadata-only response
crosses the relay sink fence. Main may receive source frames after its
synchronous provisional install, but they remain lease-private and cannot
project until provider/claim/exit validation commits that lease. Recovery
remains fenced until all recovery `pty.data` and completion drain.
project. A validated reconnect contract transfers that lease into the
attempt's private recovery quarantine; final commit, model, and desktop
admission remain fenced until all recovery `pty.data` and the exact
completion fence agree.
6. Data is ordered within a token. `pty.exit` follows every accepted data frame
and cannot bypass the sink gate or activation fence. Publishing exit seals
new data but does not retire an uncredited suffix.
@@ -244,7 +248,8 @@ renderer-only cap.
no open span, token, writer callback, drain waiter, timer, cursor, or paused
PTY. Recovery-cancellation publication failure or rejected proof fails
closed by closing only the owning provider generation and disposing its
mux, provider, publishers, provisional lease, and registration once.
mux, provider, publishers, transferred activation state, and registration
once.
14. All counters are finite safe integers. A transformed frame requires a
valid `rawLength`; malformed, excessive, stale, and cross-client values
cannot create credit or crash either process.
@@ -303,6 +308,17 @@ clientGeneration, ownerGeneration, ptyIncarnation, deliveryToken)`. Each
source credit. Duplicate/closed ledger commits are no-ops; if cancellation
proof already reclaimed a reserved span, source replacement commit rejects,
rollback skips the absent span, and settle/detach prune cached mappings.
29. Exact exit seals a provisional or private-recovery receive lease before
any later same-token frame can reserve or project. Exit publication waits
for the held contiguous prefix to admit or for cancellation proof whose
sent end covers every privately observed exact-token frame.
30. Transfer does not consume rollback authority until the ledger proves the
lease is still current and installs the private recovery sink. A stale
transfer can still retire and cancel only its own token.
31. Canceled-frame suppression retains at most one latest ordered delivery
identity per relay PTY. Transport ordering makes earlier token records
obsolete when a later cancellation is observed; exact next activation,
PTY exit, or provider teardown clears the remaining record.
## Protocol
@@ -354,6 +370,11 @@ failures without widening the approved topology:
| Drain resumed the main mux's old ordinary FIFO ahead of control | Select liveness, control, and ordinary from separate FIFO lanes; prioritize control after drain and force one ordinary write after four controls. |
| Same-turn source data escaped a provisional activation | Queue data privately under the provisional lease; commit publishes in order, while rollback drops it, restores only the active predecessor, and awaits exact cancellation proof. |
| Exit cancellation proof reclaimed spans before projection transfer | Acquire proof first, transfer projections, commit proof, prepare/finalize once, and fence every post-proof step against provider-generation close. |
| Exit during private recovery could publish before held frames settled | Seal the private lease immediately, wake the exact recovery fence, then either admit the ordered prefix before exit or retire it only after exact token-cancellation proof. |
| A superseded transfer lost its cancellation authority | Keep the outer lease unsettled until transfer ownership is verified; a stale transfer can still roll back and request cancellation for its exact token. |
| Recovery proof watermark skipped rejected or post-restore frames | Observe every exact-token private frame before retention or restore checks and reject any cancellation proof whose sent end is below that immutable high-water. |
| Canceled-token tombstones grew once per token | Retain only the latest ordered canceled token per relay PTY, replace it on later cancellation, and clear it at exact next-activation, PTY-exit, or provider-teardown boundaries. |
| Closed-generation telemetry omitted active gaps | Compact exact closed ranges and expose the count of allocated but unclosed provider generations below the closed high-water. |
The common state machine accepts only:
@@ -749,16 +770,25 @@ synchronous `beforeResolve(result)` hook in their pending-request record.
`handleResponse` parses the whole response and provisionally installs
`sourceActivation` before resolving the Promise. The lease captures the exact
prior receive state, candidate token, and a private ordered list of same-turn
source frames. Only after operation-ID result, incarnation, claimed-owner,
attach identity, stale-attempt, and spawn/attach-exit validation succeeds does
the provider commit it, publish the held frames in order, and update
`livePtyIds`. Any failure first retires the candidate and drops its held data,
restores only the nearest non-retired predecessor, then waits for a valid
`pty.cancelDelivery` response from the originating mux. It cannot erase a
newer revision or cancel a newer token. An identical already-committed
activation is an idempotent no-op; an identical provisional activation is
stale. Fresh-spawn rollback that cannot prove cancellation surfaces
`execution_owner_unavailable` rather than claiming the old token retired.
source frames. Spawn and non-recovery attach commit only after operation-ID
result, incarnation, claimed-owner, attach identity, stale-attempt, and exit
validation. Reconnect first validates the owner, incarnation, token,
checkpoint, and recovery end, installs the exact private quarantine, then
transfers the lease to its private sink before awaiting the body/completion
fence. That transfer cannot update ordinary provider listeners or
`livePtyIds`, and it cannot reach the model, runtime, desktop, or remote
consumers. Final commit switches future exact-token frames to ordinary
publication only after the body, fence, model admission, and activation
validation succeed. Any failure before transfer
retires the candidate and drops its held data, restores only the nearest
non-retired predecessor, then waits for a valid `pty.cancelDelivery` response
from the originating mux. After transfer, recovery cancellation or
provider-generation close owns cleanup; rollback is no longer authoritative.
It cannot erase a newer revision or cancel a newer token. An identical
already-committed activation is an idempotent no-op; an identical provisional
activation is stale. Fresh-spawn rollback that cannot prove cancellation
surfaces `execution_owner_unavailable` rather than claiming the old token
retired.
The receive cursor starts at `checkpointSourceEndSu`, not
`recoveryEndSu`. Source-bearing notifications for the installed identity may
@@ -767,6 +797,14 @@ quarantined as recovery; later frames stay held until the exact completion
fence commits. Unknown or mismatched source identities are rejected before
`livePtyIds`, runtime, renderer, or source obligations change.
Every exact-token private frame advances a monotonic observed-source high-water
before quarantine retention or `restoreRequired` checks. If exact exit arrives,
main wakes the recovery fence and seals the private lease immediately. It may
publish exit only after the held contiguous prefix is admitted, or after
`pty.cancelDelivery` proves `sentEndSu` covers that high-water and the accepted
checkpoint remains the credited end. Capacity-rejected and post-restore frames
therefore cannot disappear below the cancellation proof.
Output produced while a token is activating is retained by its bounded shared
cursor and can pause the native PTY; it is never sent early. Output arriving
after main's synchronous install but before authority commit is bounded by the
@@ -853,8 +891,10 @@ cancellation or local proof application rejects, the session fails closed:
it closes the exact provider generation with
`ssh_source_recovery_cancellation_failed`, disposes that generation's mux,
provider, ACK/cancellation publishers, provisional lease, and registration
once, and reconnects without clearing physical PTY ownership. If the attempt
is already stale, it performs no cleanup against the replacement. A fresh
once when still provisional, or its transferred activation state after
quarantine ownership begins, and reconnects without clearing physical PTY
ownership. If the attempt is already stale, it performs no cleanup against the
replacement. A fresh
relay that no longer retains the cached owner returns typed error `-32041`;
main clears only the cached owner/checkpoints and retries `pty.openClient`
exactly once without `resume`.
@@ -1067,7 +1107,8 @@ again, then closes the projection. This order prevents proof commit from
reclaiming a span before its published projection transfers.
If provider-generation close wins while proof is pending, it rejects and
removes the barrier plus its cancellation/preparation ownership; the late
removes the barrier plus its preparation ownership; cancellation ownership is
operation-local rather than retained in a process-lifetime tombstone. The late
proof performs no transfer, commit, preparation, final exit, or projection
close. If cancellation publication or proof fails while the barrier remains
current, main closes the provider so generation cleanup proves cancellation;
@@ -1075,6 +1116,13 @@ unrelated PTYs are not torn down on the ordinary successful timeout path. A
late emulator callback cannot mutate the reset generation. Thus relay sink
ordering and main asynchronous model ordering are both proved.
Provider generations come from one process-wide monotonic allocator and are
never reused. Main rejects closed generations through sorted, merged inclusive
ranges rather than one retained entry per reconnect. Sequential closes compact
to one range; out-of-order closes preserve exact live-generation gaps until
those generations close, then merge. This keeps stale-generation rejection
exact while retained range count follows live gaps instead of process uptime.
## Dispatcher and transport drain
Every dispatcher client is constructed with one bidirectional transport:
@@ -1502,32 +1550,35 @@ explicit/connection-close proof. The transaction creates the replacement
owner before marking source `transferring`; only exact recovery-complete
coverage terminally transfers it.
| Event | Mandatory transition |
| ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| spawn/attach response decoded | install provisional `sourceActivation` in synchronous `beforeResolve`; hold same-turn data privately, commit/publish after validation, or drop/restore and await exact cancellation proof |
| invalid-checkpoint `restoreRequired` | keep the canceled relay record through response settlement; retire only the still-current record, wake capacity, and permit a fresh-token retry |
| model accepts; desktop/mobile parses | settle that consumer |
| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker |
| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting |
| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains |
| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof |
| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore |
| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage |
| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range |
| empty or non-empty recovery completion | require checkpoint-to-`recoveryEndSu` continuity, retain that live-start anchor after activation, then admit only an exactly contiguous live frame |
| recovery completion lacks writer capacity | retain range/fence state and retry on capacity; keep at most one admitted completion attempt |
| stale/overlapping recovery attempt | cancel only that attempt's token; never mutate the replacement mux, checkpoint, PTY ownership, physical process, or owner lease |
| current recovery cancellation publication/proof fails | close the exact provider generation and release its mux, provider, publishers, provisional lease, and registration once; retain physical PTY ownership for reconnect |
| partial remote cumulative byte ACK | advance/release exact byte credit; retain the covering source frame and obligation until a complete recorded boundary |
| contiguous higher-generation source token | keep the stream attached, bind later frames to the new identity, and preserve unsettled older-token frame mappings |
| remote snapshot replacement | reserve immutable span IDs/ranges, commit only after current-generation `SnapshotEnd` coverage, otherwise roll back |
| snapshot commit after proof reclaimed a span | reject commit without dereference; rollback removes the exact reservation and skips absent spans; later settle/detach prunes cached mappings |
| remote ACK-pending overflow snapshot | reserve covered spans, publish, commit exact snapshot identity/sequence, then trim; rollback and detach on failure |
| late remote detach after token cancellation | prune already-reclaimed mappings and detach idempotently without dereference or new credit |
| exit timeout receives cancellation proof | while generation-current, transfer projections, commit proof, prepare/finalize once, and close projection with generation checks between mutations |
| provider generation closes during exit proof | reject/remove barrier ownership; a late proof cannot commit, prepare, finalize, or publish exit |
| explicit live-token reset | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport |
| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof |
| Event | Mandatory transition |
| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| spawn/attach response decoded | install provisional `sourceActivation` in synchronous `beforeResolve`; hold same-turn data privately; spawn commits after full operation validation, while reconnect transfers into an exact private recovery quarantine after contract validation and commits after fence/admission |
| invalid-checkpoint `restoreRequired` | keep the canceled relay record through response settlement; retire only the still-current record, wake capacity, and permit a fresh-token retry |
| model accepts; desktop/mobile parses | settle that consumer |
| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker |
| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting |
| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains |
| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof |
| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore |
| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage |
| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range |
| empty or non-empty recovery completion | require checkpoint-to-`recoveryEndSu` continuity, retain that live-start anchor after activation, then admit only an exactly contiguous live frame |
| exact exit during private recovery | wake the fence, reject later same-token admission, and publish exit only after the held prefix is admitted or exact cancellation proof retires it |
| recovery completion lacks writer capacity | retain range/fence state and retry on capacity; keep at most one admitted completion attempt |
| stale/overlapping recovery attempt | cancel only that attempt's token; never mutate the replacement mux, checkpoint, PTY ownership, physical process, or owner lease |
| current recovery cancellation publication/proof fails | after quarantine ownership transfer, close the exact provider generation and release its mux, provider, publishers, and registration once; retain physical PTY ownership for reconnect |
| partial remote cumulative byte ACK | advance/release exact byte credit; retain the covering source frame and obligation until a complete recorded boundary |
| contiguous higher-generation source token | keep the stream attached, bind later frames to the new identity, and preserve unsettled older-token frame mappings |
| remote snapshot replacement | reserve immutable span IDs/ranges, commit only after current-generation `SnapshotEnd` coverage, otherwise roll back |
| snapshot commit after proof reclaimed a span | reject commit without dereference; rollback removes the exact reservation and skips absent spans; later settle/detach prunes cached mappings |
| remote ACK-pending overflow snapshot | reserve covered spans, publish, commit exact snapshot identity/sequence, then trim; rollback and detach on failure |
| late remote detach after token cancellation | prune already-reclaimed mappings and detach idempotently without dereference or new credit |
| exit timeout receives cancellation proof | while generation-current, transfer projections, commit proof, prepare/finalize once, and close projection with generation checks between mutations |
| provider generation closes during exit proof | reject/remove barrier ownership; a late proof cannot commit, prepare, finalize, or publish exit |
| provider generations close over process lifetime | merge exact closed ranges; preserve out-of-order live gaps and reject every closed generation without one tombstone per reconnect |
| source tokens cancel repeatedly for one PTY | retain only the latest ordered canceled token for that PTY; replace at the next cancellation and clear at the next activation, PTY exit, or provider teardown |
| explicit live-token reset | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport |
| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof |
There is no “abandon live token” transition. A main-side path that cannot prove
settlement, transfer, or relay cancellation must keep the ledger or close the
@@ -1603,15 +1654,18 @@ main token span:
main receive token:
unseen --beforeResolve--> provisional(cursor = checkpointSourceEndSu, heldData = [])
provisional --same-turn contiguous data--> provisional(append heldData; no projection)
provisional --all spawn/attach/exit validation-->
receiving-activation(commit lease, publish heldData in order)
provisional --validation failure-->
provisional --all spawn/non-recovery attach/exit validation-->
active(commit lease, publish heldData in order)
provisional --validated recovery contract + private quarantine installed-->
receiving-activation(transfer lease to private sink, route heldData by range)
provisional --pre-transfer validation failure-->
rollback(drop heldData, restore active predecessor, request exact cancellation)
rollback --valid cancellation response--> prior exact state
rollback --publication/proof rejects--> fail operation(execution owner unavailable)
receiving-activation --validated response/no recovery--> active
receiving-activation --quarantine exact checkpoint..recoveryEnd + complete fence-->
active(expectedLiveStart = recoveryEnd)
active(final commit, expectedLiveStart = recoveryEnd)
receiving-activation --exact exit-->
exit-sealed(admit held contiguous prefix or prove exact cancellation first)
active --first/next live starts at expectedLiveStart--> active(advance expectedLiveStart)
receiving-activation|active --gap/overlap/bad fence--> canceling-only-this-token
active --pty.exit--> exit-sealed
@@ -1626,6 +1680,18 @@ main receive token:
acquiring-exit-cancellation-proof --proof cannot publish while current--> closing-provider
receiving-activation|active --overflow/cancel/close--> closed
main provider-generation guard:
open(g) --close--> closed-ranges(add [g, g], merge adjacent/overlapping)
event(g in closed-ranges) --> reject(stale generation)
event(g in an unclosed gap) --> validate PTY incarnation normally
diagnostics --> closed range count + allocated-unclosed IDs below high-water
main canceled-source retirement:
none(pty) --cancel token T--> latest(pty, T)
latest(pty, T) --later ordered cancel U--> latest(pty, U)
latest(pty, T) --frame T--> drop
latest(pty, T) --next activation/PTY exit/provider teardown--> none(pty)
remote source ledger:
frame-open --partial cumulative byte ACK-->
frame-open(release exact byte delta; retain source mapping)
@@ -1930,6 +1996,12 @@ transition:
- roll back uncommitted receive-activation leases to their exact prior
active predecessor, drop lease-private data, and await cancellation of only
the provisional token through its originating mux;
- after a validated recovery contract transfers the lease, retain held frames
only in that attempt's bounded quarantine and let cancellation proof or
provider-generation close own failure cleanup;
- on exact exit, seal the private lease, wake its fence, and either admit the
held contiguous prefix or validate cancellation against the highest exact
frame observed before retention/restore decisions;
- fail closed on the exact provider generation when a current recovery cannot
publish cancellation or validate its proof; provisional spawn rollback
failure instead surfaces execution ownership as unavailable;
@@ -1951,6 +2023,8 @@ transition:
frames, and prune cached mappings whose authoritative spans were reclaimed;
- reject and remove exit barriers plus cancellation/preparation ownership on
provider-generation close so late proofs cannot publish final exit;
- replace the latest canceled-token record per PTY on ordered cancellation and
clear it on exact next activation, PTY exit, or provider teardown;
- release shared spans no longer referenced;
- recompute native PTY pause state;
- expire pending RPC ownership.
@@ -1993,6 +2067,12 @@ the candidate mux, token, checkpoint, and `recoveryEndSu` until the exact
completion fence validates. It commits no partial body to the model. On
failure it requests cancellation only through the candidate mux and accepts
only matching proof; a stale attempt may not touch current mux state. A
quarantine-only attempt may not yet have a main-intake token identity, so
`SshRelaySession` first requires `canceled: true`, finite safe cumulative
ends, `creditedEndSu` equal to the recovery checkpoint, and `sentEndSu`
covering the highest exact-token source range observed privately. Only then is
the intake's no-identity application an intentional no-op; malformed,
under-covering, or checkpoint-mismatched proof fails closed. A
current attempt whose cancellation publication or proof rejects closes the
exact provider generation and its registered resources once, rather than
leaving ambiguous authority. After a successful empty recovery it keeps
@@ -2143,6 +2223,7 @@ logging terminal contents or raw PTY IDs:
callback duration, read-pause duration, liveness-timeout suppression,
header-ACK timestamp depth/cap denials, and keepalive coalescing;
- exit-barrier timeout count;
- closed provider-generation range count and active gaps;
- reconnect attach wave latency, per-PTY retry/failure, token supersession,
relay-initiated cancellation proof, and time-to-last-success;
- same-build negotiated and disabled-by-main sessions, unsupported manual
@@ -2306,10 +2387,20 @@ boundaries even though they ship in one PR.
control first after drain, and guarantee ordinary progress after four
controls.
- Hold provisional activation data until provider/claim/exit validation
commits; rollback drops it, restores only the active predecessor, and awaits
exact cancellation proof.
commits for spawn, or until a validated reconnect contract installs its
private quarantine; rollback owns pre-transfer failure, while cancellation
proof or generation close owns post-transfer failure.
- Acquire exit cancellation proof before projection transfer and proof commit;
prepare/finalize once and fence every post-proof step by provider generation.
- Keep timeout cancellation authority operation-local and compact monotonic
closed provider generations into exact merged ranges without weakening stale
event rejection.
- Seal private recovery on exact exit, retain rollback authority through a
stale transfer, and include every exact-token private frame in the
cancellation-proof watermark.
- Replace process-lifetime per-token blocking state with one latest ordered
cancellation record per relay PTY, cleared only at an exact lifecycle
boundary.
### Slice 5f: remaining topology validation and adapters — deliberately deferred
@@ -2341,7 +2432,7 @@ recorded evidence include:
- current-head restore-response retirement, fail-closed recovery cancellation,
partial remote ACK, higher-generation token rotation, reclaimed-span
idempotence, three-lane mux fairness, provisional claim gating, and
generation-fenced exit proof ordering;
generation-fenced exit proof ordering plus bounded exact generation closure;
- the four-test Docker OpenSSH/deployed-relay suite for source-window plateau,
concurrent typing, fixed-size filesystem/Git churn, and owner reconnect.
@@ -2366,12 +2457,23 @@ churn was 148.1/161.1 ms with 93 bulk reads; owner reconnect completed in
15.7 seconds. These measurements are direct SSH/deployed Linux relay evidence
only.
The Docker results predate the final adversarial lifecycle reconciliation and
are not current-head proof for those races. The current-head deterministic
commands below cover the integrated fixes; a direct Docker rerun and all other
live topologies remain separate.
The four-case Docker results predate the final adversarial lifecycle
reconciliation and are not current-head proof for those races. On the earlier
working tree, a deterministic test reproduced recovery completion becoming
visible at source unit 4 while lease-held recovery ended at 8. After the
validated contract began private quarantine ownership before awaiting the
body, seven provider/session files passed 94 tests and the isolated deployed
Linux relay/direct-SSH reconnect case passed in 17.7 seconds with 256 recovery
frames plus post-reconnect terminal/filesystem proof. Against the same
private-transfer artifact, the final four-case run passed in 1.1 minutes:
direct typing was 6.2/7.4 ms median/worst; ACK-stalled typing was 5.5/16.3 ms
at exactly 262,144 held source units; fixed-size filesystem/Git churn was
141.2/151.5 ms with 95 bulk reads; reconnect completed in 16.0 seconds. All
other live topologies remain separate.
At current HEAD, this focused command passed 11 files and 134 tests:
At the current working tree, the source/intake slice passed 12 files and 205
tests, the provider/session slice passed 16 files and 192 tests, and this
complementary lifecycle slice passed 13 files and 150 tests:
```bash
pnpm exec vitest run --config config/vitest.config.ts \
@@ -2382,6 +2484,8 @@ pnpm exec vitest run --config config/vitest.config.ts \
src/main/ssh/ssh-multiplexer-transport-writer.test.ts \
src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts \
src/main/providers/ssh-pty-notification-routing.test.ts \
src/main/providers/ssh-pty-source-delivery-ledger.test.ts \
src/main/ssh/ssh-pty-retired-source-deliveries.test.ts \
src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts \
src/main/providers/ssh-pty-provider-exit-race.test.ts \
src/main/ipc/ssh-pty-output-exit-deadline.test.ts \
@@ -2390,12 +2494,17 @@ pnpm exec vitest run --config config/vitest.config.ts \
The deterministic recovery seam specifically covers an empty recovery followed
by a gapped live frame, overlapping reconnect attempts, late frames after
cancel, typed stale-owner retry, response-settled `restoreRequired` retirement,
and fail-closed exact-generation cleanup. It also asserts that a proved
token-local failure causes no physical PTY shutdown, ownership delete, or
lease expiry. The remote-consumer seam covers partial cumulative ACK, safe
higher-generation token rotation, and token cancellation followed by late
detach/replacement commit, proving reclaimed span IDs are pruned idempotently.
cancel, exact exit during private recovery, stale-transfer rollback authority,
capacity-rejected and post-restore proof watermarks, typed stale-owner retry,
response-settled `restoreRequired` retirement, completion arriving before
lease-held body frames, and fail-closed exact-generation cleanup. A separate
10,000-rotation oracle proves canceled-token retirement remains one record per
PTY. Transferred recovery data remains private until body/fence validation,
and a proved token-local failure causes no physical PTY shutdown, ownership
delete, or lease expiry.
The remote-consumer seam covers partial cumulative ACK, safe higher-generation
token rotation, and token cancellation followed by late detach/replacement
commit, proving reclaimed span IDs are pruned idempotently.
### Normative unit and property matrix
@@ -2444,10 +2553,12 @@ detach/replacement commit, proving reclaimed span IDs are pruned idempotently.
producer-owned `pty.data` rather than control responses;
- feed response, recovery, completion fence, and first live data in one decoder
turn; prove synchronous `beforeResolve` installs a provisional cursor at
`checkpointSourceEndSu`, same-turn data remains unprojected until all
provider/claim/exit validation commits the lease, rollback drops held data,
restores only the active predecessor, awaits exact token-cancellation proof,
and exact gap spans reach the model once through `recoveryEndSu`;
`checkpointSourceEndSu`; deliver completion before lease-held frames, then
prove validated reconnect transfers those frames only into private
quarantine, routes ranges through `recoveryEndSu`, and projects them once
only after exact body/fence validation. Pre-transfer rollback drops held
data, restores only the active predecessor, and awaits exact
token-cancellation proof;
- reject wrong-incarnation checkpoints and checkpoints below old credit or
beyond retained live end; require a token-free `restoreRequired` response
with no partial recovery or forward clamping; retain the canceled record
@@ -2501,8 +2612,8 @@ detach/replacement commit, proving reclaimed span IDs are pruned idempotently.
gap ingestion;
- reject recovery cancellation publication and local proof application while
the attempt is current; prove the exact provider generation, mux, provider,
publishers, provisional lease, and registration close once, while unrelated
generations and physical PTY ownership remain untouched;
publishers, transferred activation, and registration close once, while
unrelated generations and physical PTY ownership remain untouched;
- prove renderer ACK/heal/write-off on V1 settles/transfers via display ranges
and emits no legacy `acknowledgeDataEvent` wire delta;
- admit interleaved spans from two token generations and prove every desktop
@@ -2554,6 +2665,10 @@ detach/replacement commit, proving reclaimed span IDs are pruned idempotently.
generation callbacks; close the generation while proof is pending and prove
no final exit publishes, then separately make proof fail while current and
prove only then the provider closes;
- close 2,048 sequential provider generations and prove one retained exact
closed range rejects early, middle, and latest stale events; close out of
order and prove an unclosed older generation remains admissible until its own
close merges the gap;
- verify round-robin progress across 50 continuously active PTYs;
- verify spans are freed only after all divergent cursors advance, cancel, or
transfer;
@@ -2670,8 +2785,9 @@ Implementation and rollout stages:
local-provider, and folder-workspace evidence remains future work before
those topologies are marked covered.
4. Current-head deterministic reconciliation and focused integrated-fix checks
are complete. Full validation and a direct Docker rerun of the final
lifecycle diff remain required before promotion.
are complete. The isolated reconnect and final four-case direct-SSH Docker
runs, full repository typecheck, and full lint/reliability suite are green;
final exact-head review remains before merge readiness.
5. Run an internal same-build canary comparing gate-off and V1 sink,
model-admission, latency, sealed-exit, recovery, and reconnect metrics while
separately counting orphan prior-version daemons;
src/main/ipc/ssh-pty-closed-generation-ranges.ts
+53
View File
@@ -0,0 +1,53 @@
type ClosedGenerationRange = {
start: number
end: number
}
export class SshPtyClosedGenerationRanges {
private readonly ranges: ClosedGenerationRange[] = []
add(generation: number): void {
let index = 0
while (index < this.ranges.length && this.ranges[index]!.end + 1 < generation) {
index++
}
const current = this.ranges[index]
if (!current || generation + 1 < current.start) {
this.ranges.splice(index, 0, { start: generation, end: generation })
return
}
current.start = Math.min(current.start, generation)
current.end = Math.max(current.end, generation)
const next = this.ranges[index + 1]
if (next && current.end + 1 >= next.start) {
current.end = Math.max(current.end, next.end)
this.ranges.splice(index + 1, 1)
}
}
has(generation: number): boolean {
for (const range of this.ranges) {
if (generation < range.start) {
return false
}
if (generation <= range.end) {
return true
}
}
return false
}
get size(): number {
return this.ranges.length
}
get activeGaps(): number {
const highWater = this.ranges.at(-1)?.end ?? 0
let closedGenerations = 0
for (const range of this.ranges) {
closedGenerations += range.end - range.start + 1
}
// Why: provider generations allocate from 1, so unclosed IDs below high-water remain active.
return highWater - closedGenerations
}
}
src/main/ipc/ssh-pty-output-exit-deadline.test.ts
+2
View File
@@ -106,6 +106,8 @@ describe('SshPtyOutputExitDeadline', () => {
source: { openedTokens: 0, ptyIdentities: 0 },
exitBarriers: 0
})
await vi.advanceTimersByTimeAsync(0)
expect(harness.exits).toEqual(['exit'])
expect(() =>
harness.intake
.getRemoteSourceRangeConsumerHooks()
src/main/ipc/ssh-pty-output-exit-deadline.ts
+10 -14
View File
@@ -21,7 +21,6 @@ type SshPtyOutputExitDeadlineDependencies = Readonly<{
export class SshPtyOutputExitDeadline {
private readonly barriersByGeneration = new Map<number, Set<SshPtyExitBarrier>>()
private readonly cancellationOwnedExits = new Set<string>()
private readonly preparedExits = new Set<string>()
private readonly preparedExitReleases = new Map<string, () => void>()
private readonly barrierMs: number
@@ -32,10 +31,18 @@ export class SshPtyOutputExitDeadline {
this.cancellationProofMs = dependencies.cancellationProofMs ?? 10_000
}
wait(event: SshPtyOutputExitEvent, promise: Promise<void>): Promise<void> {
wait(
event: SshPtyOutputExitEvent,
start: (validateNormalExit: () => void) => Promise<void>
): Promise<void> {
return new Promise((resolve, reject) => {
let timeoutStarted = false
let settled = false
const validateNormalExit = (): void => {
if (timeoutStarted) {
throw outputIntakeError('ssh_exit_delivery_canceled')
}
}
const settle = (result: { ok: true } | { ok: false; error: Error }): void => {
if (settled) {
return
@@ -53,7 +60,6 @@ export class SshPtyOutputExitDeadline {
const barrier: SshPtyExitBarrier = {
timer: setTimeout(() => {
timeoutStarted = true
this.cancellationOwnedExits.add(this.exitKey(event))
void this.cancelTimedOutExit(event, barrier).then(
() => settle({ ok: true }),
(error) => {
@@ -77,6 +83,7 @@ export class SshPtyOutputExitDeadline {
this.barriersByGeneration.set(event.providerGeneration, barriers)
}
barriers.add(barrier)
const promise = start(validateNormalExit)
void promise.then(
() => {
if (!timeoutStarted) {
@@ -102,11 +109,6 @@ export class SshPtyOutputExitDeadline {
this.barriersByGeneration.delete(providerGeneration)
}
const prefix = `${providerGeneration}\0`
for (const key of this.cancellationOwnedExits) {
if (key.startsWith(prefix)) {
this.cancellationOwnedExits.delete(key)
}
}
for (const key of this.preparedExits) {
if (key.startsWith(prefix)) {
this.releasePreparedExitKey(key)
@@ -115,12 +117,6 @@ export class SshPtyOutputExitDeadline {
}
}
validateNormalExit(event: SshPtyOutputExitEvent): void {
if (this.cancellationOwnedExits.has(this.exitKey(event))) {
throw outputIntakeError('ssh_exit_delivery_canceled')
}
}
prepareExitOnce(event: SshPtyOutputExitEvent): void {
const key = this.exitKey(event)
if (this.preparedExits.has(key)) {
src/main/ipc/ssh-pty-output-generation-guard.test.ts
+46
View File
@@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest'
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
const event = (providerGeneration: number) => ({
id: 'pty-1',
providerGeneration,
ptyIncarnation: 'incarnation-1'
})
describe('SshPtyOutputGenerationGuard', () => {
it('compacts sequential closures without weakening stale rejection', () => {
const guard = new SshPtyOutputGenerationGuard(() => false)
for (let generation = 1; generation <= 2_048; generation++) {
guard.closeGeneration(generation)
}
expect(guard.getDebugSnapshot()).toEqual({
closedRanges: 1,
activeGaps: 0,
activePtys: 0,
sealedPtys: 0
})
expect(() => guard.validate(event(1_024))).toThrow('ssh_output_stale_generation')
expect(() => guard.validate(event(2_049))).not.toThrow()
})
it('counts and merges out-of-order live generation gaps exactly', () => {
const guard = new SshPtyOutputGenerationGuard(() => false)
guard.closeGeneration(4)
guard.closeGeneration(2)
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 2, activeGaps: 2 })
guard.closeGeneration(3)
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 1, activeGaps: 1 })
expect(() => guard.validate(event(1))).not.toThrow()
expect(() => guard.validate(event(2))).toThrow('ssh_output_stale_generation')
guard.closeGeneration(1)
expect(guard.getDebugSnapshot()).toEqual({
closedRanges: 1,
activeGaps: 0,
activePtys: 0,
sealedPtys: 0
})
})
})
src/main/ipc/ssh-pty-output-generation-guard.ts
+11 -1
View File
@@ -4,12 +4,13 @@ import {
sshPtyGenerationKey,
validOutputLength
} from './ssh-pty-output-intake-validation'
import { SshPtyClosedGenerationRanges } from './ssh-pty-closed-generation-ranges'
export class SshPtyOutputGenerationGuard {
private readonly latestGenerationByPty = new Map<string, number>()
private readonly incarnationByPty = new Map<string, string>()
private readonly sealedPtys = new Set<string>()
private readonly closedGenerations = new Set<number>()
private readonly closedGenerations = new SshPtyClosedGenerationRanges()
constructor(private readonly isDisposed: () => boolean) {}
@@ -94,4 +95,13 @@ export class SshPtyOutputGenerationGuard {
activeGenerations(): ReadonlySet<number> {
return new Set(this.latestGenerationByPty.values())
}
getDebugSnapshot() {
return {
closedRanges: this.closedGenerations.size,
activeGaps: this.closedGenerations.activeGaps,
activePtys: this.latestGenerationByPty.size,
sealedPtys: this.sealedPtys.size
}
}
}
src/main/ipc/ssh-pty-output-intake-registry.ts
+11
View File
@@ -121,6 +121,17 @@ export function applySshPtySourceCancellationProof(
return installedIntake?.applySourceCancellationProof(event, proof) ?? false
}
export function applySshPtySourceRecoveryCancellationProof(
event: SshPtyOutputExitEvent,
proof: SshPtySourceCancellationProof
): boolean {
if (!installedIntake) {
return false
}
installedIntake.applySourceRecoveryCancellationProof(event, proof)
return true
}
function outputIntakeUnavailableError(): Error {
return Object.assign(new Error('ssh_output_intake_unavailable'), {
code: 'ssh_output_intake_unavailable'
src/main/ipc/ssh-pty-output-intake.test.ts
+53
View File
@@ -654,6 +654,59 @@ describe('SshPtyOutputIntake', () => {
await expect(sibling).resolves.toMatchObject({ ptyId: 'pty-2' })
})
it('accepts recovery cancellation proof before any replacement span is admitted', () => {
const harness = createHarness()
expect(() =>
harness.intake.applySourceRecoveryCancellationProof(
{
id: 'pty-1',
code: -1,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
},
{ sentEndSu: 8, creditedEndSu: 4 }
)
).not.toThrow()
expect(harness.intake.getDebugSnapshot().source).toEqual({
openedTokens: 0,
ptyIdentities: 0
})
})
it('reclaims a partially admitted recovery prefix from authoritative proof', async () => {
const harness = createHarness()
const receipt = harness.intake.acceptData(
event({
source: {
spanId: 'recovery-span',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'recovery-token',
sourceStartSu: 4,
sourceEndSu: 8
}
})
)
harness.completions[0]!.resolve()
await receipt
harness.intake.applySourceRecoveryCancellationProof(
{
id: 'pty-1',
code: -1,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
},
{ sentEndSu: 12, creditedEndSu: 4 }
)
expect(harness.intake.getDebugSnapshot().source).toEqual({
openedTokens: 0,
ptyIdentities: 0
})
})
it('rejects late same-generation data after ordered exit cleanup', async () => {
const harness = createHarness({}, { exitBarrierMs: 1000 })
const first = harness.intake.acceptData(event())
src/main/ipc/ssh-pty-output-intake.ts
+16 -3
View File
@@ -162,10 +162,15 @@ export class SshPtyOutputIntake {
async acceptExit(event: SshPtyOutputExitEvent): Promise<void> {
this.generationGuard.sealExit(event)
await this.exitDeadline.wait(event, this.finishExit(event))
await this.exitDeadline.wait(event, (validateNormalExit) =>
this.finishExit(event, validateNormalExit)
)
}
private async finishExit(event: SshPtyOutputExitEvent): Promise<void> {
private async finishExit(
event: SshPtyOutputExitEvent,
validateNormalExit: () => void
): Promise<void> {
await settleSshPtyOutputExit({
event,
admission: this.admission,
@@ -173,7 +178,7 @@ export class SshPtyOutputIntake {
dependencies: this.dependencies,
validateGeneration: () => {
this.generationGuard.validate(event)
this.exitDeadline.validateNormalExit(event)
validateNormalExit()
},
prepareExit: () => this.exitDeadline.prepareExitOnce(event),
afterAdmissionIdle: () => this.sourceObligations.sealPty(event),
@@ -248,11 +253,19 @@ export class SshPtyOutputIntake {
return this.sourceObligations.applyCancellationProof(event, proof)
}
applySourceRecoveryCancellationProof(
event: SshPtyOutputExitEvent,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
this.sourceObligations.applyRecoveryCancellationProof(event, proof)
}
getDebugSnapshot() {
return {
model: this.admission.getDebugSnapshot(),
projection: this.projections.getDebugSnapshot(),
source: this.sourceObligations.getDebugSnapshot(),
generation: this.generationGuard.getDebugSnapshot(),
exitBarriers: this.exitDeadline.activeBarriers
}
}
src/main/ipc/ssh-pty-output-source-obligations.ts
+10
View File
@@ -179,6 +179,16 @@ export class SshPtyOutputSourceObligations {
return true
}
applyRecoveryCancellationProof(
event: SshPtyOutputExitEvent,
proof: SshPtySourceCancellationProof
): void {
const identity = this.identityByPty.get(this.ptyKey(event))
if (identity) {
this.coordinator.applyRecoveryCancellationProof(identity, proof)
}
}
closeGeneration(providerGeneration: number, reason: string): void {
this.remoteConsumers.closeGeneration(providerGeneration, reason)
this.coordinator.closeGeneration(providerGeneration, reason)
src/main/ipc/ssh-pty-source-obligation-coordinator.ts
+11
View File
@@ -160,6 +160,17 @@ export class SshPtySourceObligationCoordinator {
)
}
applyRecoveryCancellationProof(
identity: PtySourceDeliveryIdentity,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
this.ledger.applyRecoveryCancellationProof(identity, proof)
this.rejectWaiters(
(group) => samePtySourceDelivery(group.identity, identity),
new Error('ssh_source_delivery_canceled')
)
}
closeGeneration(providerGeneration: number, reason: string): number {
this.rejectWaiters(
(group) => group.identity.providerGeneration === providerGeneration,
src/main/ipc/ssh-pty-source-obligation-ledger.test.ts
+25
View File
@@ -237,6 +237,31 @@ describe('SshPtySourceObligationLedger', () => {
)
})
it('applies recovery cancellation proof over a locally admitted prefix', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner, 4)
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 12, creditedEndSu: 4 })
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
})
it('rejects recovery cancellation proof that misses local intake state', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner, 4)
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
expect(() =>
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 7, creditedEndSu: 4 })
).toThrow('invalid')
expect(() =>
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 8, creditedEndSu: 5 })
).toThrow('invalid')
})
it('bounds closed-token tombstones and removes committed reservation indexes', () => {
const ledger = new SshPtySourceObligationLedger()
const owners = Array.from({ length: 300 }, (_, index) =>
src/main/ipc/ssh-pty-source-obligation-ledger.ts
+10
View File
@@ -34,6 +34,7 @@ import {
type TokenRecord
} from './ssh-pty-source-obligation-state'
import {
applySourceRecoveryCancellationProof,
cancelSourceObligationTransfer,
commitSourceObligationTransfer,
modelAcceptedSourceEnd,
@@ -223,6 +224,15 @@ export class SshPtySourceObligationLedger {
this.closeToken(token)
}
applyRecoveryCancellationProof(
identity: PtySourceDeliveryIdentity,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
const token = this.requireToken(identity)
applySourceRecoveryCancellationProof(token, proof)
this.closeToken(token)
}
closeGeneration(providerGeneration: number, reason: string): number {
return closeSourceGeneration(
this.tokens,
src/main/ipc/ssh-pty-source-obligation-transitions.ts
+16
View File
@@ -4,10 +4,26 @@ import type {
} from './ssh-pty-source-obligation-contract'
import {
advanceSourceTerminalEnd,
cancelOpenSourceObligations,
requireSourceSpan,
type TokenRecord
} from './ssh-pty-source-obligation-state'
export function applySourceRecoveryCancellationProof(
token: TokenRecord,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
if (
token.state !== 'active' ||
proof.sentEndSu < token.receivedEndSu ||
proof.creditedEndSu !== token.ackPublishedEndSu ||
proof.creditedEndSu > token.receivedEndSu
) {
throw new Error('SSH PTY source recovery cancellation proof is stale or invalid')
}
cancelOpenSourceObligations(token, 'relay-recovery-cancellation-proof')
}
export function transitionOpenSourceObligation(
spanOwners: ReadonlyMap<string, TokenRecord>,
spanId: string,
src/main/ipc/ssh.test.ts
+1
View File
@@ -114,6 +114,7 @@ vi.mock('./ssh-pty-output-intake-registry', () => ({
return () => ++generation
})(),
applySshPtySourceCancellationProof: vi.fn().mockReturnValue(false),
applySshPtySourceRecoveryCancellationProof: vi.fn().mockReturnValue(false),
closeSshPtyOutputGeneration: vi.fn(),
getSshPtyAcceptedSourceCheckpoints: vi.fn().mockReturnValue([]),
installSshPtySourceAckPublisher: vi.fn().mockReturnValue(() => {}),
src/main/providers/ssh-pty-notification-routing.test.ts
+137
View File
@@ -508,6 +508,143 @@ describe('subscribeSshPtyNotifications', () => {
)
})
it('routes held and later recovery frames only to the private sink until commit', () => {
const { handler, dataListeners, livePtyIds, installReceivingActivation } = createSubscription()
const onData = vi.fn()
const onRecoveryData = vi.fn()
dataListeners.add(onData)
const lease = installReceivingActivation(
'pty-1',
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 12 })
)
const publishSource = (data: string, sourceEndSu: number): void => {
handler('pty.data', {
id: 'pty-1',
data,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu,
sourceLengthSu: 4
})
}
publishSource('held', 8)
const recoveryLease = lease.transferToRecovery(onRecoveryData)
publishSource('next', 12)
expect(onRecoveryData.mock.calls.map(([payload]) => payload.data)).toEqual(['held', 'next'])
expect(onData).not.toHaveBeenCalled()
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
recoveryLease.commit()
expect(onData).not.toHaveBeenCalled()
publishSource('live', 16)
expect(onRecoveryData).toHaveBeenCalledTimes(2)
expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'live' }))
expect(livePtyIds).toContain('ssh:conn@@pty-1')
})
it('retires an exited private recovery when its activation commits', () => {
const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } =
createSubscription()
const onData = vi.fn()
const onRecoveryData = vi.fn()
dataListeners.add(onData)
const lease = installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 4 }))
handler('pty.data', {
id: 'pty-1',
data: 'held',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 4,
sourceLengthSu: 4
})
const recoveryLease = lease.transferToRecovery(onRecoveryData)
handler('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' })
recoveryLease.commit()
handler('pty.data', {
id: 'pty-1',
data: 'late',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 8,
sourceLengthSu: 4
})
expect(onRecoveryData).toHaveBeenCalledOnce()
expect(onData).not.toHaveBeenCalled()
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
expect(mux.request).toHaveBeenCalledWith(
'pty.cancelDelivery',
expect.objectContaining({ id: 'pty-1', deliveryToken: 'token-1' })
)
})
it('retires private recovery locally and restores the exact predecessor', () => {
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
const onRecoveryData = vi.fn()
dataListeners.add(onData)
installReceivingActivation(
'pty-1',
sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 })
).commit()
handler('pty.data', {
id: 'pty-1',
data: 'pre',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3,
sourceLengthSu: 3
})
const replacement = installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-new',
checkpointSourceEndSu: 3,
recoveryEndSu: 6
})
)
handler('pty.data', {
id: 'pty-1',
data: 'new',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
clientGeneration: 3,
ownerGeneration: 4,
sourceEndSu: 6,
sourceLengthSu: 3
})
replacement.transferToRecovery(onRecoveryData).retire()
handler('pty.data', {
id: 'pty-1',
data: 'old',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 6,
sourceLengthSu: 3
})
expect(onRecoveryData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' }))
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old'])
expect(mux.request).not.toHaveBeenCalled()
})
it('rejects a stale activation without disturbing current continuity', () => {
const { handler, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
src/main/providers/ssh-pty-notification-routing.ts
+37 -19
View File
@@ -9,12 +9,19 @@ import type {
import { parseSshPtySourceFrame } from './ssh-pty-source-frame'
import {
SshPtySourceDeliveryLedger,
type PendingSshPtySourceData,
type SshPtyReceivingActivationLease
type PendingSshPtySourceData
} from './ssh-pty-source-delivery-ledger'
export type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback }
export type { SshPtyReceivingActivationLease } from './ssh-pty-source-delivery-ledger'
export type SshPtyRecoveryActivationLease = Readonly<{
commit: () => void
retire: () => void
}>
export type SshPtyReceivingActivationLease = Readonly<{
commit: () => void
rollback: () => Promise<boolean>
transferToRecovery: (sink: SshPtyDataCallback) => SshPtyRecoveryActivationLease
}>
export type SshPtyNotificationSubscription = Readonly<{
dispose: () => void
@@ -35,26 +42,30 @@ export function subscribeSshPtyNotifications(args: {
providerGeneration: number
resolvePtyIncarnation: (relayPtyId: string, incarnationId?: unknown) => string
}): SshPtyNotificationSubscription {
const publishData = (pending: PendingSshPtySourceData): void => {
const toDataPayload = (pending: PendingSshPtySourceData): Parameters<SshPtyDataCallback>[0] => {
const id = args.toAppPtyId(pending.relayPtyId)
const ptyIncarnation = args.resolvePtyIncarnation(
pending.relayPtyId,
pending.params.ptyIncarnation ?? pending.params.incarnationId
)
args.livePtyIds.add(id)
return {
id,
data: pending.data,
providerGeneration: args.providerGeneration,
ptyIncarnation: pending.source ? (pending.params.ptyIncarnation as string) : ptyIncarnation,
...(typeof pending.params.rawLength === 'number'
? { sequenceChars: pending.params.rawLength }
: {}),
...(pending.params.transformed === true ? { transformed: true } : {}),
...(typeof pending.params.seq === 'number' ? { seq: pending.params.seq } : {}),
...(pending.source ? { source: pending.source } : {})
}
}
const publishData = (pending: PendingSshPtySourceData): void => {
const payload = toDataPayload(pending)
args.livePtyIds.add(payload.id)
for (const listener of args.dataListeners) {
listener({
id,
data: pending.data,
providerGeneration: args.providerGeneration,
ptyIncarnation: pending.source ? (pending.params.ptyIncarnation as string) : ptyIncarnation,
...(typeof pending.params.rawLength === 'number'
? { sequenceChars: pending.params.rawLength }
: {}),
...(pending.params.transformed === true ? { transformed: true } : {}),
...(typeof pending.params.seq === 'number' ? { seq: pending.params.seq } : {}),
...(pending.source ? { source: pending.source } : {})
})
listener(payload)
}
}
const sourceDeliveries = new SshPtySourceDeliveryLedger(args.mux, publishData)
@@ -118,8 +129,15 @@ export function subscribeSshPtyNotifications(args: {
})
return Object.freeze({
dispose,
installReceivingActivation: (relayPtyId, activation) =>
sourceDeliveries.install(relayPtyId, activation)
installReceivingActivation: (relayPtyId, activation) => {
const lease = sourceDeliveries.install(relayPtyId, activation)
return Object.freeze({
commit: lease.commit,
rollback: lease.rollback,
transferToRecovery: (sink: SshPtyDataCallback) =>
lease.transferToRecovery((pending) => sink(toDataPayload(pending)))
})
}
})
}
src/main/providers/ssh-pty-source-delivery-ledger.test.ts
+44
View File
@@ -0,0 +1,44 @@
import { describe, expect, it, vi } from 'vitest'
import { SshPtySourceDeliveryLedger } from './ssh-pty-source-delivery-ledger'
describe('SshPtySourceDeliveryLedger', () => {
it('retains cancellation ownership when recovery transfer is superseded', async () => {
const request = vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
const ledger = new SshPtySourceDeliveryLedger({ request } as never, vi.fn())
const older = ledger.install(
'pty-1',
Object.freeze({
status: 'pending',
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
checkpointSourceEndSu: 0,
recoveryEndSu: 0
})
)
ledger.install(
'pty-1',
Object.freeze({
status: 'pending',
clientGeneration: 3,
ownerGeneration: 4,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
checkpointSourceEndSu: 0,
recoveryEndSu: 0
})
)
expect(() => older.transferToRecovery(vi.fn())).toThrow('ssh_source_receiving_activation_stale')
await expect(older.rollback()).resolves.toBe(true)
expect(request).toHaveBeenCalledOnce()
expect(request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-old'
})
})
})
src/main/providers/ssh-pty-source-delivery-ledger.ts
+100 -21
View File
@@ -10,8 +10,10 @@ export type PendingSshPtySourceData = Readonly<{
}>
type SourceDeliveryLeaseState = {
phase: 'provisional' | 'committing' | 'committed' | 'retired'
phase: 'provisional' | 'recovery' | 'committing' | 'committed' | 'retired'
pendingData: PendingSshPtySourceData[]
recoverySink?: (pending: PendingSshPtySourceData) => void
exited: boolean
}
type SourceDeliveryState = Readonly<{
@@ -21,9 +23,17 @@ type SourceDeliveryState = Readonly<{
previous?: SourceDeliveryState
}>
export type SshPtyReceivingActivationLease = Readonly<{
export type SshPtyRecoveryActivationLease = Readonly<{
commit: () => void
retire: () => void
}>
export type SshPtySourceDeliveryLease = Readonly<{
commit: () => void
rollback: () => Promise<boolean>
transferToRecovery: (
sink: (pending: PendingSshPtySourceData) => void
) => SshPtyRecoveryActivationLease
}>
export class SshPtySourceDeliveryLedger {
@@ -34,10 +44,7 @@ export class SshPtySourceDeliveryLedger {
private readonly publishData: (pending: PendingSshPtySourceData) => void
) {}
install(
relayPtyId: string,
activation: PtySourceReceivingActivation
): SshPtyReceivingActivationLease {
install(relayPtyId: string, activation: PtySourceReceivingActivation): SshPtySourceDeliveryLease {
if (!relayPtyId || activation.ptyIncarnation.length === 0) {
throw new Error('ssh_source_receiving_activation_invalid')
}
@@ -69,7 +76,9 @@ export class SshPtySourceDeliveryLedger {
sourceEndSu: pending.source.sourceEndSu
}) as SourceDeliveryState
this.deliveryByPty.set(pending.relayPtyId, accepted)
if (accepted.lease.phase !== 'committed') {
if (accepted.lease.phase === 'recovery') {
accepted.lease.recoverySink?.(pending)
} else if (accepted.lease.phase !== 'committed') {
accepted.lease.pendingData.push(pending)
} else {
this.publishData(pending)
@@ -78,8 +87,11 @@ export class SshPtySourceDeliveryLedger {
}
recordExit(relayPtyId: string): void {
if (this.deliveryByPty.get(relayPtyId)?.lease.phase === 'committed') {
const current = this.deliveryByPty.get(relayPtyId)
if (current?.lease.phase === 'committed') {
this.deliveryByPty.delete(relayPtyId)
} else if (current) {
current.lease.exited = true
}
}
@@ -87,10 +99,11 @@ export class SshPtySourceDeliveryLedger {
relayPtyId: string,
activation: PtySourceReceivingActivation,
previous: SourceDeliveryState | undefined
): SshPtyReceivingActivationLease {
): SshPtySourceDeliveryLease {
const leaseState: SourceDeliveryLeaseState = {
phase: 'provisional',
pendingData: []
pendingData: [],
exited: false
}
this.deliveryByPty.set(
relayPtyId,
@@ -102,10 +115,11 @@ export class SshPtySourceDeliveryLedger {
})
)
let settled = false
let transferInProgress = false
let rollbackSettlement: Promise<boolean> | undefined
return Object.freeze({
commit: () => {
if (settled) {
if (settled || transferInProgress) {
return
}
settled = true
@@ -115,13 +129,67 @@ export class SshPtySourceDeliveryLedger {
if (rollbackSettlement) {
return rollbackSettlement
}
if (settled) {
if (settled || transferInProgress) {
return Promise.resolve(false)
}
settled = true
this.retire(relayPtyId, previous, leaseState)
rollbackSettlement = settleExactSourceDeliveryCancellation(this.mux, relayPtyId, activation)
return rollbackSettlement
},
transferToRecovery: (sink) => {
if (settled || transferInProgress || leaseState.phase !== 'provisional') {
throw new Error('ssh_source_receiving_activation_stale')
}
transferInProgress = true
try {
const recoveryLease = this.transferToRecovery(relayPtyId, leaseState, previous, sink)
settled = true
return recoveryLease
} finally {
transferInProgress = false
}
}
})
}
private transferToRecovery(
relayPtyId: string,
lease: SourceDeliveryLeaseState,
previous: SourceDeliveryState | undefined,
sink: (pending: PendingSshPtySourceData) => void
): SshPtyRecoveryActivationLease {
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
this.retire(relayPtyId, previous, lease)
throw new Error('ssh_source_receiving_activation_stale')
}
lease.phase = 'recovery'
lease.recoverySink = sink
try {
while (lease.pendingData.length > 0) {
sink(lease.pendingData.shift()!)
}
} catch (error) {
this.retire(relayPtyId, previous, lease)
throw error
}
let settled = false
return Object.freeze({
commit: () => {
if (settled) {
return
}
settled = true
lease.recoverySink = undefined
this.commit(relayPtyId, lease)
},
retire: () => {
if (settled) {
return
}
settled = true
lease.recoverySink = undefined
this.retire(relayPtyId, previous, lease)
}
})
}
@@ -138,6 +206,10 @@ export class SshPtySourceDeliveryLedger {
}
lease.phase = 'committed'
const current = this.deliveryByPty.get(relayPtyId)
if (lease.exited && current?.lease === lease) {
this.deliveryByPty.delete(relayPtyId)
return
}
if (current?.lease === lease && current.previous) {
this.deliveryByPty.set(
relayPtyId,
@@ -156,10 +228,15 @@ export class SshPtySourceDeliveryLedger {
lease: SourceDeliveryLeaseState
): void {
lease.phase = 'retired'
lease.recoverySink = undefined
lease.pendingData.splice(0)
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
return
}
if (lease.exited) {
this.deliveryByPty.delete(relayPtyId)
return
}
const predecessor = activePredecessor(previous)
if (predecessor) {
this.deliveryByPty.set(relayPtyId, predecessor)
@@ -169,18 +246,19 @@ export class SshPtySourceDeliveryLedger {
}
}
function settledReceivingActivationLease(): SshPtyReceivingActivationLease {
return Object.freeze({ commit: () => {}, rollback: async () => true })
function settledReceivingActivationLease(): SshPtySourceDeliveryLease {
return Object.freeze({
commit: () => {},
rollback: async () => true,
transferToRecovery: () => Object.freeze({ commit: () => {}, retire: () => {} })
})
}
function activePredecessor(
previous: SourceDeliveryState | undefined
): SourceDeliveryState | undefined {
let candidate = previous
while (candidate?.lease.phase === 'retired') {
candidate = candidate.previous
function activePredecessor(previous?: SourceDeliveryState): SourceDeliveryState | undefined {
while (previous?.lease.phase === 'retired') {
previous = previous.previous
}
return candidate
return previous
}
function sameReceivingActivation(
@@ -205,6 +283,7 @@ function acceptsSourceFrame(
return Boolean(
current &&
current.lease.phase !== 'retired' &&
!current.lease.exited &&
current.activation.ptyIncarnation === params.ptyIncarnation &&
current.activation.deliveryToken === source.deliveryToken &&
current.activation.clientGeneration === source.clientGeneration &&
src/main/ssh/ssh-pty-retired-source-deliveries.test.ts
+35
View File
@@ -0,0 +1,35 @@
import { describe, expect, it } from 'vitest'
import { SshPtyRetiredSourceDeliveries } from './ssh-pty-retired-source-deliveries'
const source = (deliveryToken: string, relayPtyId = 'pty-1') => ({
relayPtyId,
deliveryToken,
clientGeneration: 2,
ownerGeneration: 3
})
describe('SshPtyRetiredSourceDeliveries', () => {
it('retains only the latest canceled token for each ordered PTY stream', () => {
const retired = new SshPtyRetiredSourceDeliveries()
for (let token = 0; token < 10_000; token++) {
retired.retire(1, source(`token-${token}`))
}
expect(retired.size).toBe(1)
expect(retired.has(1, source('token-9999'))).toBe(true)
expect(retired.has(1, source('token-9998'))).toBe(false)
})
it('retires state at the next activation or PTY exit boundary', () => {
const retired = new SshPtyRetiredSourceDeliveries()
retired.retire(1, source('old-token'))
retired.retire(1, source('other-token', 'pty-2'))
retired.activate('pty-1')
expect(retired.has(1, source('old-token'))).toBe(false)
expect(retired.has(1, source('other-token', 'pty-2'))).toBe(true)
expect(retired.size).toBe(1)
})
})
src/main/ssh/ssh-pty-retired-source-deliveries.ts
+36
View File
@@ -0,0 +1,36 @@
import type { SshPtySourceFrame } from '../providers/ssh-pty-source-frame'
type SourceDeliveryIdentity = Pick<
SshPtySourceFrame,
'relayPtyId' | 'deliveryToken' | 'clientGeneration' | 'ownerGeneration'
>
export class SshPtyRetiredSourceDeliveries {
private readonly keyByRelayPtyId = new Map<string, string>()
has(providerGeneration: number, source: SourceDeliveryIdentity): boolean {
return (
this.keyByRelayPtyId.get(source.relayPtyId) === sourceDeliveryKey(providerGeneration, source)
)
}
retire(providerGeneration: number, source: SourceDeliveryIdentity): void {
this.keyByRelayPtyId.set(source.relayPtyId, sourceDeliveryKey(providerGeneration, source))
}
activate(relayPtyId: string): void {
this.keyByRelayPtyId.delete(relayPtyId)
}
clear(): void {
this.keyByRelayPtyId.clear()
}
get size(): number {
return this.keyByRelayPtyId.size
}
}
function sourceDeliveryKey(providerGeneration: number, source: SourceDeliveryIdentity): string {
return `${providerGeneration}\0${source.clientGeneration}\0${source.ownerGeneration}\0${source.deliveryToken}`
}
src/main/ssh/ssh-relay-session-data-delivery.test.ts
+49 -51
View File
@@ -40,7 +40,8 @@ vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({
getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []),
installSshPtySourceAckPublisher: vi.fn(() => () => {}),
installSshPtySourceCancellationPublisher: vi.fn(() => () => {}),
applySshPtySourceCancellationProof: vi.fn()
applySshPtySourceCancellationProof: vi.fn(),
applySshPtySourceRecoveryCancellationProof: vi.fn()
}))
vi.mock('./ssh-channel-multiplexer', () => ({
@@ -122,6 +123,8 @@ const {
} = await import('../ipc/pty')
const { closeSshPtyOutputGeneration } = await import('../ipc/ssh-pty-output-intake-registry')
const { applySshPtySourceCancellationProof } = await import('../ipc/ssh-pty-output-intake-registry')
const { applySshPtySourceRecoveryCancellationProof } =
await import('../ipc/ssh-pty-output-intake-registry')
const { getSshPtyAcceptedSourceCheckpoints } = await import('../ipc/ssh-pty-output-intake-registry')
const { installSshPtySourceAckPublisher } = await import('../ipc/ssh-pty-output-intake-registry')
const { deployAndLaunchRelay } = await import('./ssh-relay-deploy')
@@ -134,6 +137,7 @@ describe('SshRelaySession data delivery', () => {
vi.mocked(getPtyIdsForConnection).mockReturnValue([])
vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([])
vi.mocked(applySshPtySourceCancellationProof).mockReturnValue(true)
vi.mocked(applySshPtySourceRecoveryCancellationProof).mockReturnValue(true)
openConsumerSessionMock.mockImplementation(async (_mux, options) => ({
mode: 'negotiated',
clientInstanceId: options.clientInstanceId,
@@ -666,6 +670,42 @@ describe('SshRelaySession data delivery', () => {
() => vi.mocked(registerSshPtyProvider).mock.calls.at(-1)?.[1]
)
let transferDisposedMux = false
const publishHeldRecovery = (sink: (payload: unknown) => void): void => {
for (const [data, sourceStartSu, sourceEndSu] of [
['re', 4, 6],
['co', 6, 8],
['live', 8, 12]
] as const) {
sink({
id: 'ssh:target-1@@pty-1',
data,
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: sourceEndSu - sourceStartSu,
source: {
relayPtyId: 'pty-1',
spanId: `new-token:${sourceStartSu}:${sourceEndSu}`,
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu,
sourceEndSu
}
})
}
}
const recoveryActivationLease = {
commit: vi.fn(),
retire: vi.fn()
}
const sourceActivationLease = {
commit: vi.fn(),
rollback: vi.fn(async () => true),
transferToRecovery: vi.fn((sink: (payload: unknown) => void) => {
publishHeldRecovery(sink)
return recoveryActivationLease
})
}
attachForReconnectMock.mockImplementation(async () => {
const canceled = onNotificationByMethodMock.mock.calls.findLast(
([method]) => method === 'pty.deliveryCanceled'
@@ -681,38 +721,6 @@ describe('SshRelaySession data delivery', () => {
})
transferDisposedMux = muxDisposeMock.mock.calls.length !== disposeCount
queueMicrotask(() => {
ptyDataHandlerRef.current?.({
id: 'ssh:target-1@@pty-1',
data: 're',
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 2,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:4:6',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 4,
sourceEndSu: 6
}
})
ptyDataHandlerRef.current?.({
id: 'ssh:target-1@@pty-1',
data: 'co',
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 2,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:6:8',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 6,
sourceEndSu: 8
}
})
const complete = onNotificationByMethodMock.mock.calls.findLast(
([method]) => method === 'pty.recoveryComplete'
)?.[1] as ((params: Record<string, unknown>) => void) | undefined
@@ -725,22 +733,6 @@ describe('SshRelaySession data delivery', () => {
checkpointSourceEndSu: 4,
recoveryEndSu: 8
})
ptyDataHandlerRef.current?.({
id: 'ssh:target-1@@pty-1',
data: 'live',
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 4,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:8:12',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 8,
sourceEndSu: 12
}
})
})
return {
incarnationId: 'incarnation-1',
@@ -752,7 +744,8 @@ describe('SshRelaySession data delivery', () => {
deliveryToken: 'new-token',
checkpointSourceEndSu: 4,
recoveryEndSu: 8
}
},
sourceActivationLease
}
})
@@ -773,6 +766,11 @@ describe('SshRelaySession data delivery', () => {
'live'
])
expect(transferDisposedMux).toBe(false)
expect(sourceActivationLease.transferToRecovery).toHaveBeenCalledOnce()
expect(sourceActivationLease.commit).not.toHaveBeenCalled()
expect(sourceActivationLease.rollback).not.toHaveBeenCalled()
expect(recoveryActivationLease.commit).toHaveBeenCalledOnce()
expect(recoveryActivationLease.retire).not.toHaveBeenCalled()
expect(mockWindow.webContents.send).not.toHaveBeenCalledWith('pty:replay', expect.anything())
const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length
@@ -809,7 +807,7 @@ describe('SshRelaySession data delivery', () => {
ownerGeneration: 2,
deliveryToken: 'new-token'
})
expect(applySshPtySourceCancellationProof).toHaveBeenCalledWith(
expect(applySshPtySourceRecoveryCancellationProof).toHaveBeenCalledWith(
{
id: `ssh:${targetId}@@pty-1`,
code: -1,
src/main/ssh/ssh-relay-session-incarnation.test.ts
+1
View File
@@ -22,6 +22,7 @@ vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({
allocateSshPtyProviderGeneration: vi.fn(() => 31),
closeSshPtyOutputGeneration: vi.fn(),
getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []),
applySshPtySourceRecoveryCancellationProof: vi.fn(() => true),
installSshPtySourceAckPublisher: vi.fn(() => () => {})
}))
vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') }))
src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts
+1
View File
@@ -22,6 +22,7 @@ vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({
allocateSshPtyProviderGeneration: vi.fn(() => 17),
closeSshPtyOutputGeneration: vi.fn(),
getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []),
applySshPtySourceRecoveryCancellationProof: vi.fn(() => true),
installSshPtySourceAckPublisher: vi.fn(() => () => {})
}))
vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') }))
src/main/ssh/ssh-relay-session-recovery-races.test.ts
+325 -15
View File
@@ -4,6 +4,7 @@ import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixt
const {
acceptOutputDataMock,
acceptOutputExitMock,
muxRequestMock,
onNotificationByMethodMock,
openConsumerSessionMock,
@@ -12,9 +13,11 @@ const {
sourceAckCleanupMock,
sourceCancellationCleanupMock,
attachForReconnectMock,
ptyDataHandlerRef
ptyDataHandlerRef,
ptyExitHandlerRef
} = vi.hoisted(() => ({
acceptOutputDataMock: vi.fn().mockResolvedValue(undefined),
acceptOutputExitMock: vi.fn().mockResolvedValue(undefined),
muxRequestMock: vi.fn(),
onNotificationByMethodMock: vi.fn(),
openConsumerSessionMock: vi.fn(),
@@ -23,7 +26,8 @@ const {
sourceAckCleanupMock: vi.fn(),
sourceCancellationCleanupMock: vi.fn(),
attachForReconnectMock: vi.fn().mockResolvedValue({}),
ptyDataHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) }
ptyDataHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) },
ptyExitHandlerRef: { current: undefined as undefined | ((payload: unknown) => void) }
}))
vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() }))
@@ -32,13 +36,14 @@ vi.mock('./ssh-pty-consumer-session', () => ({
}))
vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({
acceptSshPtyOutputData: acceptOutputDataMock,
acceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined),
acceptSshPtyOutputExit: acceptOutputExitMock,
allocateSshPtyProviderGeneration: vi.fn(() => 23),
closeSshPtyOutputGeneration: vi.fn(),
getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []),
installSshPtySourceAckPublisher: vi.fn(() => sourceAckCleanupMock),
installSshPtySourceCancellationPublisher: vi.fn(() => sourceCancellationCleanupMock),
applySshPtySourceCancellationProof: vi.fn(() => true)
applySshPtySourceCancellationProof: vi.fn(() => true),
applySshPtySourceRecoveryCancellationProof: vi.fn(() => true)
}))
vi.mock('./ssh-channel-multiplexer', () => ({
SshChannelMultiplexer: class MockSshChannelMultiplexer {
@@ -62,7 +67,10 @@ vi.mock('../providers/ssh-pty-provider', () => ({
return () => {}
})
onReplay = vi.fn().mockReturnValue(() => {})
onExit = vi.fn().mockReturnValue(() => {})
onExit = vi.fn().mockImplementation((handler) => {
ptyExitHandlerRef.current = handler
return () => {}
})
attachForReconnect = attachForReconnectMock
setPtyDeliveryPauseAdapter = vi.fn()
dispose = ptyProviderDisposeMock
@@ -81,6 +89,7 @@ vi.mock('../ipc/pty', () => ({
unregisterSshPtyProvider: vi.fn(),
getSshPtyProvider: vi.fn().mockReturnValue({ dispose: vi.fn() }),
getPtyIdsForConnection: vi.fn().mockReturnValue([]),
isCurrentPtyExit: vi.fn(() => true),
clearPtyOwnershipForConnection: vi.fn(),
clearProviderPtyState: vi.fn(),
deletePtyOwnership: vi.fn(),
@@ -110,15 +119,19 @@ const {
const { closeSshPtyOutputGeneration, getSshPtyAcceptedSourceCheckpoints } =
await import('../ipc/ssh-pty-output-intake-registry')
const { applySshPtySourceCancellationProof } = await import('../ipc/ssh-pty-output-intake-registry')
const { applySshPtySourceRecoveryCancellationProof } =
await import('../ipc/ssh-pty-output-intake-registry')
describe('SshRelaySession recovery race fencing', () => {
beforeEach(() => {
vi.clearAllMocks()
ptyDataHandlerRef.current = undefined
ptyExitHandlerRef.current = undefined
attachForReconnectMock.mockResolvedValue({})
vi.mocked(getPtyIdsForConnection).mockReturnValue([])
vi.mocked(getSshPtyAcceptedSourceCheckpoints).mockReturnValue([])
vi.mocked(applySshPtySourceCancellationProof).mockReturnValue(true)
vi.mocked(applySshPtySourceRecoveryCancellationProof).mockReturnValue(true)
muxRequestMock.mockResolvedValue([])
mockDeploySuccess()
})
@@ -191,6 +204,197 @@ describe('SshRelaySession recovery race fencing', () => {
return { session, deps }
}
it('publishes held recovery data before an exact exit without waiting for completion', async () => {
const targetId = 'exit-with-complete-private-body'
const { session, deps } = await prepareRecovery(targetId)
const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() }
const sourceActivationLease = {
commit: vi.fn(),
rollback: vi.fn(async () => true),
transferToRecovery: vi.fn((sink: (payload: unknown) => void) => {
sink({
id: `ssh:${targetId}@@pty-1`,
data: 'held',
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 4,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:4:8',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 4,
sourceEndSu: 8
}
})
return recoveryActivationLease
})
}
attachForReconnectMock.mockImplementation(async () => {
ptyExitHandlerRef.current?.({
id: `ssh:${targetId}@@pty-1`,
code: 0,
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
incarnationId: 'incarnation-1'
})
return {
incarnationId: 'incarnation-1',
sourceRecovery: {
status: 'pending',
clientGeneration: 2,
ownerGeneration: 2,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'new-token',
checkpointSourceEndSu: 4,
recoveryEndSu: 8
},
sourceActivationLease
}
})
await session.reconnect(deps.mockConn)
expect(acceptOutputDataMock).toHaveBeenCalledWith(expect.objectContaining({ data: 'held' }))
expect(acceptOutputExitMock).toHaveBeenCalledOnce()
expect(acceptOutputDataMock.mock.invocationCallOrder[0]).toBeLessThan(
acceptOutputExitMock.mock.invocationCallOrder[0]!
)
expect(sourceActivationLease.transferToRecovery).toHaveBeenCalledOnce()
expect(sourceActivationLease.rollback).not.toHaveBeenCalled()
expect(recoveryActivationLease.commit).toHaveBeenCalledOnce()
expect(recoveryActivationLease.retire).not.toHaveBeenCalled()
expect(muxRequestMock).not.toHaveBeenCalledWith('pty.cancelDelivery', expect.anything())
expect(setPtyOwnership).not.toHaveBeenCalled()
expect(deps.mockStore.markSshRemotePtyLease).not.toHaveBeenCalledWith(
targetId,
'pty-1',
'attached'
)
})
it('settles exact cancellation before publishing an exit with incomplete recovery data', async () => {
const targetId = 'exit-with-incomplete-private-body'
let settleCancellation: ((proof: unknown) => void) | undefined
muxRequestMock.mockImplementation((method) =>
method === 'pty.cancelDelivery'
? new Promise((resolve) => {
settleCancellation = resolve
})
: Promise.resolve([])
)
const { session, deps } = await prepareRecovery(targetId)
const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() }
const sourceActivationLease = {
commit: vi.fn(),
rollback: vi.fn(async () => true),
transferToRecovery: vi.fn((sink: (payload: unknown) => void) => {
sink({
id: `ssh:${targetId}@@pty-1`,
data: 'partial',
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 2,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:4:6',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 4,
sourceEndSu: 6
}
})
return recoveryActivationLease
})
}
attachForReconnectMock.mockImplementation(async () => {
ptyExitHandlerRef.current?.({
id: `ssh:${targetId}@@pty-1`,
code: 0,
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
incarnationId: 'incarnation-1'
})
return {
incarnationId: 'incarnation-1',
sourceRecovery: {
status: 'pending',
clientGeneration: 2,
ownerGeneration: 2,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'new-token',
checkpointSourceEndSu: 4,
recoveryEndSu: 8
},
sourceActivationLease
}
})
const reconnect = session.reconnect(deps.mockConn)
await vi.waitFor(() =>
expect(muxRequestMock).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token'
})
)
expect(acceptOutputExitMock).not.toHaveBeenCalled()
settleCancellation?.({ canceled: true, sentEndSu: 6, creditedEndSu: 4 })
await reconnect
expect(acceptOutputDataMock).not.toHaveBeenCalled()
expect(acceptOutputExitMock).toHaveBeenCalledOnce()
expect(recoveryActivationLease.commit).not.toHaveBeenCalled()
expect(recoveryActivationLease.retire).toHaveBeenCalledOnce()
expect(sourceActivationLease.rollback).not.toHaveBeenCalled()
expect(applySshPtySourceRecoveryCancellationProof).toHaveBeenCalledOnce()
})
it('awaits provisional cancellation proof before publishing an exact recovery exit', async () => {
const targetId = 'exit-before-recovery-identity'
let settleRollback: ((settled: boolean) => void) | undefined
const { session, deps } = await prepareRecovery(targetId)
const sourceActivationLease = {
commit: vi.fn(),
rollback: vi.fn(
() =>
new Promise<boolean>((resolve) => {
settleRollback = resolve
})
),
transferToRecovery: vi.fn()
}
attachForReconnectMock.mockImplementation(async () => {
ptyExitHandlerRef.current?.({
id: `ssh:${targetId}@@pty-1`,
code: 0,
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
incarnationId: 'incarnation-1'
})
return {
incarnationId: 'incarnation-1',
sourceRecovery: { status: 'restoreRequired', reason: 'checkpointUnavailable' },
sourceActivationLease
}
})
const reconnect = session.reconnect(deps.mockConn)
await vi.waitFor(() => expect(sourceActivationLease.rollback).toHaveBeenCalledOnce())
expect(acceptOutputExitMock).not.toHaveBeenCalled()
settleRollback?.(true)
await reconnect
expect(acceptOutputExitMock).toHaveBeenCalledOnce()
expect(sourceActivationLease.transferToRecovery).not.toHaveBeenCalled()
expect(sourceActivationLease.commit).not.toHaveBeenCalled()
})
it('retains the empty recovery end as the first post-activation source anchor', async () => {
const targetId = 'empty-recovery-gap'
const { session, deps } = await prepareRecovery(targetId)
@@ -306,9 +510,91 @@ describe('SshRelaySession recovery race fencing', () => {
expect(muxDisposeMock).not.toHaveBeenCalledWith('shutdown')
})
it.each(['publication', 'proof'] as const)(
it('rejects cancellation proof below capacity-rejected and later private frames', async () => {
const targetId = 'capacity-watermark'
muxRequestMock.mockImplementation(async (method) =>
method === 'pty.cancelDelivery' ? { canceled: true, sentEndSu: 8, creditedEndSu: 4 } : []
)
const { session, deps } = await prepareRecovery(targetId)
const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() }
const sourceActivationLease = {
commit: vi.fn(),
rollback: vi.fn(),
transferToRecovery: vi.fn((sink: (payload: unknown) => void) => {
sink({
id: `ssh:${targetId}@@pty-1`,
data: 'x'.repeat(2 * 1024 * 1024 + 1),
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 4,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:4:8',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 4,
sourceEndSu: 8
}
})
sink({
id: `ssh:${targetId}@@pty-1`,
data: 'later',
providerGeneration: 23,
ptyIncarnation: 'incarnation-1',
sequenceChars: 4,
source: {
relayPtyId: 'pty-1',
spanId: 'new-token:8:12',
clientGeneration: 2,
ownerGeneration: 2,
deliveryToken: 'new-token',
sourceStartSu: 8,
sourceEndSu: 12
}
})
return recoveryActivationLease
})
}
attachForReconnectMock.mockResolvedValue({
incarnationId: 'incarnation-1',
sourceRecovery: {
status: 'pending',
clientGeneration: 2,
ownerGeneration: 2,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'new-token',
checkpointSourceEndSu: 4,
recoveryEndSu: 12
},
sourceActivationLease
})
await session.reconnect(deps.mockConn)
expect(muxRequestMock).toHaveBeenCalledWith(
'pty.cancelDelivery',
expect.objectContaining({ deliveryToken: 'new-token' })
)
expect(vi.mocked(closeSshPtyOutputGeneration).mock.calls).toContainEqual([
23,
'ssh_source_recovery_cancellation_failed'
])
expect(applySshPtySourceRecoveryCancellationProof).not.toHaveBeenCalled()
expect(recoveryActivationLease.commit).not.toHaveBeenCalled()
expect(recoveryActivationLease.retire).toHaveBeenCalledOnce()
})
it.each([
['publication', undefined],
['proof', { canceled: true, sentEndSu: 8, creditedEndSu: 4 }],
['negative-end', { canceled: true, sentEndSu: -1, creditedEndSu: 4 }],
['credited-ahead', { canceled: true, sentEndSu: 8, creditedEndSu: 9 }],
['checkpoint-mismatch', { canceled: true, sentEndSu: 8, creditedEndSu: 3 }],
['sent-before-private-end', { canceled: true, sentEndSu: 7, creditedEndSu: 4 }]
] as const)(
'closes one provider generation when recovery cancellation %s rejects',
async (failure) => {
async (failure, cancellationResult) => {
const targetId = `cancel-${failure}-failure`
muxRequestMock.mockImplementation(async (method) => {
if (method !== 'pty.cancelDelivery') {
@@ -317,16 +603,21 @@ describe('SshRelaySession recovery race fencing', () => {
if (failure === 'publication') {
throw new Error('cancel publication failed')
}
return { canceled: true, sentEndSu: 8, creditedEndSu: 4 }
return cancellationResult
})
if (failure === 'proof') {
vi.mocked(applySshPtySourceCancellationProof).mockImplementation(() => {
vi.mocked(applySshPtySourceRecoveryCancellationProof).mockImplementation(() => {
throw new Error('cancel proof rejected')
})
}
const { session, deps } = await prepareRecovery(targetId)
const onRelayLost = vi.fn()
const activationLease = { commit: vi.fn(), rollback: vi.fn() }
const recoveryActivationLease = { commit: vi.fn(), retire: vi.fn() }
const activationLease = {
commit: vi.fn(),
rollback: vi.fn(),
transferToRecovery: vi.fn(() => recoveryActivationLease)
}
session.setOnRelayLost(onRelayLost)
let cleanupCountsBeforeFailure:
| {
@@ -388,8 +679,11 @@ describe('SshRelaySession recovery race fencing', () => {
expect(sourceCancellationCleanupMock.mock.calls).toHaveLength(before.cancellation + 1)
expect(vi.mocked(unregisterSshPtyProvider).mock.calls).toHaveLength(before.unregister + 1)
expect(unregisterSshPtyProvider).toHaveBeenLastCalledWith(targetId)
expect(activationLease.rollback).toHaveBeenCalledOnce()
expect(activationLease.transferToRecovery).toHaveBeenCalledOnce()
expect(activationLease.commit).not.toHaveBeenCalled()
expect(activationLease.rollback).not.toHaveBeenCalled()
expect(recoveryActivationLease.commit).not.toHaveBeenCalled()
expect(recoveryActivationLease.retire).toHaveBeenCalledOnce()
expect(onRelayLost).toHaveBeenCalledOnce()
expect(session.getState()).toBe('reconnecting')
expect(clearProviderPtyState).not.toHaveBeenCalled()
@@ -412,8 +706,18 @@ describe('SshRelaySession recovery race fencing', () => {
it('keeps a stale overlapping recovery from canceling or mutating its replacement', async () => {
const targetId = 'overlapping-recovery'
const { session, deps } = await prepareRecovery(targetId)
const staleLease = { commit: vi.fn(), rollback: vi.fn() }
const replacementLease = { commit: vi.fn(), rollback: vi.fn() }
const staleRecoveryLease = { commit: vi.fn(), retire: vi.fn() }
const replacementRecoveryLease = { commit: vi.fn(), retire: vi.fn() }
const staleLease = {
commit: vi.fn(),
rollback: vi.fn(),
transferToRecovery: vi.fn(() => staleRecoveryLease)
}
const replacementLease = {
commit: vi.fn(),
rollback: vi.fn(),
transferToRecovery: vi.fn(() => replacementRecoveryLease)
}
attachForReconnectMock.mockImplementation(async () => {
const ownerGeneration = openConsumerSessionMock.mock.calls.length
if (ownerGeneration === 3) {
@@ -466,10 +770,16 @@ describe('SshRelaySession recovery race fencing', () => {
expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledTimes(1)
expect(deps.mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(targetId, 'pty-1', 'attached')
expect(setPtyOwnership).toHaveBeenCalledTimes(1)
expect(staleLease.rollback).toHaveBeenCalledOnce()
expect(staleLease.transferToRecovery).toHaveBeenCalledOnce()
expect(staleLease.commit).not.toHaveBeenCalled()
expect(replacementLease.commit).toHaveBeenCalledOnce()
expect(staleLease.rollback).not.toHaveBeenCalled()
expect(staleRecoveryLease.commit).not.toHaveBeenCalled()
expect(staleRecoveryLease.retire).toHaveBeenCalledOnce()
expect(replacementLease.transferToRecovery).toHaveBeenCalledOnce()
expect(replacementLease.commit).not.toHaveBeenCalled()
expect(replacementLease.rollback).not.toHaveBeenCalled()
expect(replacementRecoveryLease.commit).toHaveBeenCalledOnce()
expect(replacementRecoveryLease.retire).not.toHaveBeenCalled()
expect(clearProviderPtyState).not.toHaveBeenCalled()
expect(clearPtyOwnershipForConnection).not.toHaveBeenCalled()
expect(deletePtyOwnership).not.toHaveBeenCalled()
src/main/ssh/ssh-relay-session.test.ts
+1
View File
@@ -31,6 +31,7 @@ vi.mock('../ipc/ssh-pty-output-intake-registry', () => ({
allocateSshPtyProviderGeneration: vi.fn(() => 41),
closeSshPtyOutputGeneration: vi.fn(),
getSshPtyAcceptedSourceCheckpoints: vi.fn(() => []),
applySshPtySourceRecoveryCancellationProof: vi.fn(() => true),
installSshPtySourceAckPublisher: vi.fn(() => () => {})
}))
src/main/ssh/ssh-relay-session.ts
+189 -80
View File
@@ -11,6 +11,7 @@ import { SshChannelMultiplexer } from './ssh-channel-multiplexer'
import { SshPtyProvider } from '../providers/ssh-pty-provider'
import type { SshPtyAttachResult } from '../providers/ssh-pty-session-reattach'
import type { SshPtyDataCallback, SshPtyExitCallback } from '../providers/ssh-pty-provider-contract'
import type { SshPtyRecoveryActivationLease } from '../providers/ssh-pty-notification-routing'
import { isSshPtyIdentityMismatchError, isSshPtyNotFoundError } from '../providers/ssh-pty-errors'
import { toAppSshPtyId, toRelaySshPtyId } from '../providers/ssh-pty-id'
import { SshFilesystemProvider } from '../providers/ssh-filesystem-provider'
@@ -43,6 +44,7 @@ import {
acceptSshPtyOutputExit,
allocateSshPtyProviderGeneration,
applySshPtySourceCancellationProof,
applySshPtySourceRecoveryCancellationProof,
closeSshPtyOutputGeneration,
getSshPtyAcceptedSourceCheckpoints,
installSshPtySourceAckPublisher,
@@ -89,6 +91,7 @@ import type {
PtySourceRecoveryRequest
} from '../../shared/pty-source-recovery-contract'
import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget'
import { SshPtyRetiredSourceDeliveries } from './ssh-pty-retired-source-deliveries'
export type RelaySessionState = 'idle' | 'deploying' | 'ready' | 'reconnecting' | 'disposed'
@@ -111,6 +114,7 @@ type PendingPtyReattach = {
recovery?: PtySourceRecoveryPending
recoveryComplete?: PtySourceRecoveryComplete
nextRecoverySourceSu?: number
highestRecoverySourceEndSu?: number
replacementDeliveryToken?: string
restoreRequired?: string
recoveryWaiters: Set<() => void>
@@ -272,7 +276,7 @@ export class SshRelaySession {
nextSourceSu?: number
}>
>()
private readonly blockedSourceDeliveryKeys = new Set<string>()
private readonly retiredSourceDeliveries = new SshPtyRetiredSourceDeliveries()
private readonly ptyConsumerClientInstanceId: string
private ptyConsumerSessionState: SshPtyConsumerSessionState | null = null
@@ -1189,7 +1193,7 @@ export class SshRelaySession {
unregisterSshFilesystemProvider(this.targetId)
unregisterSshGitProvider(this.targetId)
this.sourceIdentityByRelayPtyId.clear()
this.blockedSourceDeliveryKeys.clear()
this.retiredSourceDeliveries.clear()
for (const pending of this.pendingPtyReattaches.values()) {
for (const resolve of pending.recoveryWaiters) {
resolve()
@@ -1313,6 +1317,7 @@ export class SshRelaySession {
if (pendingReattach && !pendingReattach.activated) {
// Why: attach response and exit can share one transport batch, before incarnation restoration runs.
pendingReattach.exits.push(payload)
this.wakeRecovery(pendingReattach)
return
}
if (!isCurrentPtyExit(payload)) {
@@ -1327,9 +1332,7 @@ export class SshRelaySession {
const offeredSource = payload.source
if (
offeredSource &&
this.blockedSourceDeliveryKeys.has(
this.sourceDeliveryKey(payload.providerGeneration, offeredSource)
)
this.retiredSourceDeliveries.has(payload.providerGeneration, offeredSource)
) {
return Promise.resolve()
}
@@ -1388,6 +1391,7 @@ export class SshRelaySession {
}
private quarantineReattachData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void {
this.observePrivateRecoveryFrame(pending, payload)
if (pending.restoreRequired) {
return
}
@@ -1399,17 +1403,47 @@ export class SshRelaySession {
this.wakeRecovery(pending)
return
}
if (pending.recoveryComplete) {
pending.liveData.push(payload)
return
}
this.routeQuarantinedReattachData(pending, payload)
}
private routeQuarantinedReattachData(
pending: PendingPtyReattach,
payload: SshPtyDataPayload
): void {
this.observePrivateRecoveryFrame(pending, payload)
if (!pending.recovery) {
pending.queuedData.push(payload)
return
}
if (
pending.recoveryComplete &&
pending.nextRecoverySourceSu === pending.recovery.recoveryEndSu
) {
pending.liveData.push(payload)
return
}
this.admitRecoveryData(pending, payload)
}
private observePrivateRecoveryFrame(
pending: PendingPtyReattach,
payload: SshPtyDataPayload
): void {
const recovery = pending.recovery
if (
recovery &&
payload.source?.deliveryToken === recovery.deliveryToken &&
payload.source.clientGeneration === recovery.clientGeneration &&
payload.source.ownerGeneration === recovery.ownerGeneration &&
payload.ptyIncarnation === recovery.ptyIncarnation
) {
pending.highestRecoverySourceEndSu = Math.max(
pending.highestRecoverySourceEndSu ?? recovery.checkpointSourceEndSu,
payload.source.sourceEndSu
)
}
}
private admitRecoveryData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void {
if (pending.restoreRequired) {
return
@@ -1531,12 +1565,10 @@ export class SshRelaySession {
creditedEndSu: params.creditedEndSu as number
}
)
this.blockedSourceDeliveryKeys.add(
this.sourceDeliveryKey(generation, {
relayPtyId: id,
...identity
})
)
this.retiredSourceDeliveries.retire(generation, {
relayPtyId: id,
...identity
})
this.sourceIdentityByRelayPtyId.delete(id)
} catch {
/* Invalid proof retains the active token identity. */
@@ -1567,6 +1599,7 @@ export class SshRelaySession {
private retireExitedPty(payload: SshPtyExitPayload, deliveryHandled = false): void {
const relayPtyId = toRelaySshPtyId(this.targetId, payload.id)
this.retiredSourceDeliveries.activate(relayPtyId)
clearProviderPtyState(payload.id)
deletePtyOwnership(payload.id)
ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.delete(payload.id)
@@ -1693,6 +1726,7 @@ export class SshRelaySession {
}
this.pendingPtyReattaches.set(appPtyId, pendingReattach)
let sourceActivationLease: SshPtyAttachResult['sourceActivationLease']
let recoveryActivationLease: SshPtyRecoveryActivationLease | undefined
try {
const recoveryRequest = this.sourceRecoveryRequest(appPtyId)
const attachResult = await this.attachPtyWithRetry(
@@ -1712,7 +1746,7 @@ export class SshRelaySession {
!attachResult.incarnationId ||
exit.incarnationId === attachResult.incarnationId
)
if (exitDuringAttach) {
if (exitDuringAttach && !recoveryRequest) {
if (attachResult.incarnationId) {
restorePtyIncarnation(appPtyId, attachResult.incarnationId)
this.runtime?.acceptPtyIncarnationForExit(appPtyId, attachResult.incarnationId)
@@ -1720,18 +1754,61 @@ export class SshRelaySession {
await this.acceptPtyExit(exitDuringAttach)
return
}
if (
recoveryRequest &&
!(await this.finishSourceRecovery(
if (recoveryRequest) {
const recovered = await this.finishSourceRecovery(
ptyId,
appPtyId,
attachResult,
recoveryRequest,
pendingReattach,
shouldContinue
))
) {
return
shouldContinue,
() => {
const lease = sourceActivationLease
if (!lease) {
return
}
recoveryActivationLease = lease.transferToRecovery((payload) =>
this.quarantineReattachData(pendingReattach, payload)
)
sourceActivationLease = undefined
}
)
if (!recovered) {
const recoveryExit = this.findExactPendingExit(
pendingReattach,
attachResult.incarnationId
)
if (
recoveryExit &&
shouldContinue() &&
this.ownsPtyRecoveryAttempt(appPtyId, pendingReattach)
) {
if (recoveryActivationLease) {
recoveryActivationLease.retire()
recoveryActivationLease = undefined
} else if (sourceActivationLease) {
const canceled = await sourceActivationLease.rollback()
sourceActivationLease = undefined
if (!canceled) {
throw sourceRecoveryCancellationError(
new Error('ssh_source_activation_cancellation_unproven')
)
}
}
this.preparePtyIncarnationForExit(appPtyId, attachResult.incarnationId)
await this.acceptPtyExit(recoveryExit)
}
return
}
const recoveryExit = this.findExactPendingExit(pendingReattach, attachResult.incarnationId)
if (recoveryExit) {
this.preparePtyIncarnationForExit(appPtyId, attachResult.incarnationId)
pendingReattach.activated = true
recoveryActivationLease?.commit()
recoveryActivationLease = undefined
await this.acceptPtyExit(recoveryExit)
return
}
}
if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pendingReattach)) {
return
@@ -1747,6 +1824,8 @@ export class SshRelaySession {
}
this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached')
pendingReattach.activated = true
recoveryActivationLease?.commit()
recoveryActivationLease = undefined
const exitAfterActivation = pendingReattach.exits.find(
(exit) =>
!exit.incarnationId ||
@@ -1771,6 +1850,7 @@ export class SshRelaySession {
}
this.handlePtyReattachFailure(ptyId, appPtyId, pendingReattach, error)
} finally {
recoveryActivationLease?.retire()
sourceActivationLease?.rollback()
if (this.pendingPtyReattaches.get(appPtyId) === pendingReattach) {
this.pendingPtyReattaches.delete(appPtyId)
@@ -1779,6 +1859,28 @@ export class SshRelaySession {
}
}
private findExactPendingExit(
pending: PendingPtyReattach,
ptyIncarnation: string | undefined
): SshPtyExitPayload | undefined {
if (!ptyIncarnation) {
return undefined
}
return pending.exits.find(
(exit) =>
exit.providerGeneration === pending.providerGeneration &&
exit.ptyIncarnation === ptyIncarnation
)
}
private preparePtyIncarnationForExit(appPtyId: string, ptyIncarnation: string | undefined): void {
if (!ptyIncarnation) {
return
}
restorePtyIncarnation(appPtyId, ptyIncarnation)
this.runtime?.acceptPtyIncarnationForExit(appPtyId, ptyIncarnation)
}
private restoreReattachedPtyRuntime(
appPtyId: string,
incarnationId: string,
@@ -1949,21 +2051,23 @@ export class SshRelaySession {
attachResult: SshPtyAttachResult,
request: PtySourceRecoveryRequest,
pending: PendingPtyReattach,
shouldContinue: () => boolean
shouldContinue: () => boolean,
activateRecoveryQuarantine: () => void
): Promise<boolean> {
const recovery = attachResult.sourceRecovery
const pendingRecovery = recovery?.status === 'pending' ? recovery : undefined
const owner = this.negotiatedPtyConsumerOwner()
if (
!owner?.outputFlowControl ||
recovery?.status !== 'pending' ||
!pendingRecovery ||
request.status !== 'checkpoint' ||
recovery.clientGeneration !== owner.clientGeneration ||
recovery.ownerGeneration !== owner.ownerGeneration ||
recovery.ptyIncarnation !== attachResult.incarnationId ||
recovery.ptyIncarnation !== request.ptyIncarnation ||
recovery.checkpointSourceEndSu !== request.acceptedSourceEndSu ||
pendingRecovery.clientGeneration !== owner.clientGeneration ||
pendingRecovery.ownerGeneration !== owner.ownerGeneration ||
pendingRecovery.ptyIncarnation !== attachResult.incarnationId ||
pendingRecovery.ptyIncarnation !== request.ptyIncarnation ||
pendingRecovery.checkpointSourceEndSu !== request.acceptedSourceEndSu ||
(pending.replacementDeliveryToken !== undefined &&
pending.replacementDeliveryToken !== recovery.deliveryToken)
pending.replacementDeliveryToken !== pendingRecovery.deliveryToken)
) {
if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) {
return false
@@ -1971,31 +2075,35 @@ export class SshRelaySession {
await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending)
return false
}
pending.recovery = recovery
pending.nextRecoverySourceSu = recovery.checkpointSourceEndSu
const acceptedRecovery = pendingRecovery
pending.recovery = acceptedRecovery
pending.nextRecoverySourceSu = acceptedRecovery.checkpointSourceEndSu
this.retiredSourceDeliveries.activate(relayPtyId)
this.sourceIdentityByRelayPtyId.set(relayPtyId, {
deliveryToken: recovery.deliveryToken,
clientGeneration: recovery.clientGeneration,
ownerGeneration: recovery.ownerGeneration,
ptyIncarnation: recovery.ptyIncarnation,
nextSourceSu: recovery.checkpointSourceEndSu
deliveryToken: acceptedRecovery.deliveryToken,
clientGeneration: acceptedRecovery.clientGeneration,
ownerGeneration: acceptedRecovery.ownerGeneration,
ptyIncarnation: acceptedRecovery.ptyIncarnation,
nextSourceSu: acceptedRecovery.checkpointSourceEndSu
})
activateRecoveryQuarantine()
for (const payload of pending.queuedData.splice(0)) {
this.admitRecoveryData(pending, payload)
this.routeQuarantinedReattachData(pending, payload)
}
await this.waitForRecoveryFence(pending, shouldContinue)
const complete = pending.recoveryComplete
const exactExit = this.findExactPendingExit(pending, acceptedRecovery.ptyIncarnation)
const complete = pending.recoveryComplete ?? (exactExit ? acceptedRecovery : undefined)
if (
!shouldContinue() ||
pending.restoreRequired ||
!complete ||
complete.deliveryToken !== recovery.deliveryToken ||
complete.clientGeneration !== recovery.clientGeneration ||
complete.ownerGeneration !== recovery.ownerGeneration ||
complete.ptyIncarnation !== recovery.ptyIncarnation ||
complete.checkpointSourceEndSu !== recovery.checkpointSourceEndSu ||
complete.recoveryEndSu !== recovery.recoveryEndSu ||
pending.nextRecoverySourceSu !== recovery.recoveryEndSu
complete.deliveryToken !== acceptedRecovery.deliveryToken ||
complete.clientGeneration !== acceptedRecovery.clientGeneration ||
complete.ownerGeneration !== acceptedRecovery.ownerGeneration ||
complete.ptyIncarnation !== acceptedRecovery.ptyIncarnation ||
complete.checkpointSourceEndSu !== acceptedRecovery.checkpointSourceEndSu ||
complete.recoveryEndSu !== acceptedRecovery.recoveryEndSu ||
pending.nextRecoverySourceSu !== acceptedRecovery.recoveryEndSu
) {
if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) {
return false
@@ -2003,16 +2111,16 @@ export class SshRelaySession {
await this.abandonPtySourceRecovery(relayPtyId, appPtyId, pending)
return false
}
let nextLiveSourceSu = recovery.recoveryEndSu
let nextLiveSourceSu = acceptedRecovery.recoveryEndSu
for (const payload of pending.liveData) {
if (
!payload.source ||
payload.source.deliveryToken !== recovery.deliveryToken ||
payload.source.clientGeneration !== recovery.clientGeneration ||
payload.source.ownerGeneration !== recovery.ownerGeneration ||
payload.source.deliveryToken !== acceptedRecovery.deliveryToken ||
payload.source.clientGeneration !== acceptedRecovery.clientGeneration ||
payload.source.ownerGeneration !== acceptedRecovery.ownerGeneration ||
payload.source.sourceStartSu !== nextLiveSourceSu ||
payload.source.sourceEndSu <= payload.source.sourceStartSu ||
payload.ptyIncarnation !== recovery.ptyIncarnation
payload.ptyIncarnation !== acceptedRecovery.ptyIncarnation
) {
if (!shouldContinue() || !this.ownsPtyRecoveryAttempt(appPtyId, pending)) {
return false
@@ -2042,17 +2150,17 @@ export class SshRelaySession {
}
const acceptedSourceEndSu = pending.liveData.reduce(
(endSu, payload) => Math.max(endSu, payload.source?.sourceEndSu ?? endSu),
recovery.recoveryEndSu
acceptedRecovery.recoveryEndSu
)
ptyConsumerRecoveryByTarget.get(this.targetId)?.checkpointsByAppPtyId.set(
relayPtyId,
Object.freeze({
id: relayPtyId,
providerGeneration: this.activePtyProviderGeneration!,
clientGeneration: recovery.clientGeneration,
ownerGeneration: recovery.ownerGeneration,
ptyIncarnation: recovery.ptyIncarnation,
deliveryToken: recovery.deliveryToken,
clientGeneration: acceptedRecovery.clientGeneration,
ownerGeneration: acceptedRecovery.ownerGeneration,
ptyIncarnation: acceptedRecovery.ptyIncarnation,
deliveryToken: acceptedRecovery.deliveryToken,
acceptedSourceEndSu
})
)
@@ -2068,6 +2176,7 @@ export class SshRelaySession {
shouldContinue() &&
!pending.recoveryComplete &&
!pending.restoreRequired &&
!this.findExactPendingExit(pending, pending.recovery?.ptyIncarnation) &&
Date.now() < deadline
) {
await new Promise<void>((resolve) => {
@@ -2086,7 +2195,11 @@ export class SshRelaySession {
pending.recoveryWaiters.add(settle)
})
}
if (!pending.recoveryComplete && !pending.restoreRequired) {
if (
!pending.recoveryComplete &&
!pending.restoreRequired &&
!this.findExactPendingExit(pending, pending.recovery?.ptyIncarnation)
) {
pending.restoreRequired = 'recoveryFenceTimeout'
}
}
@@ -2108,23 +2221,31 @@ export class SshRelaySession {
ownerGeneration: recovery.ownerGeneration,
deliveryToken: recovery.deliveryToken
}
this.blockedSourceDeliveryKeys.add(
this.sourceDeliveryKey(providerGeneration, {
relayPtyId,
deliveryToken: recovery.deliveryToken,
clientGeneration: recovery.clientGeneration,
ownerGeneration: recovery.ownerGeneration
})
)
this.retiredSourceDeliveries.retire(providerGeneration, {
relayPtyId,
deliveryToken: recovery.deliveryToken,
clientGeneration: recovery.clientGeneration,
ownerGeneration: recovery.ownerGeneration
})
try {
const result = (await mux.request('pty.cancelDelivery', cancellationRequest)) as Record<
string,
unknown
>
const highestPrivateSourceEndSu = pending.liveData.reduce(
(endSu, payload) => Math.max(endSu, payload.source?.sourceEndSu ?? endSu),
Math.max(
pending.nextRecoverySourceSu ?? recovery.checkpointSourceEndSu,
pending.highestRecoverySourceEndSu ?? recovery.checkpointSourceEndSu
)
)
if (
result.canceled !== true ||
!Number.isSafeInteger(result.sentEndSu) ||
!Number.isSafeInteger(result.creditedEndSu)
(result.sentEndSu as number) < highestPrivateSourceEndSu ||
!Number.isSafeInteger(result.creditedEndSu) ||
result.creditedEndSu !== recovery.checkpointSourceEndSu ||
(result.creditedEndSu as number) > (result.sentEndSu as number)
) {
throw new Error('ssh_source_cancellation_proof_invalid')
}
@@ -2136,7 +2257,7 @@ export class SshRelaySession {
return
}
if (identity) {
const applied = applySshPtySourceCancellationProof(
const applied = applySshPtySourceRecoveryCancellationProof(
{
id: appPtyId,
code: -1,
@@ -2206,16 +2327,4 @@ export class SshRelaySession {
left.ptyIncarnation === right.ptyIncarnation
)
}
private sourceDeliveryKey(
providerGeneration: number,
source: Readonly<{
relayPtyId?: string
deliveryToken: string
clientGeneration: number
ownerGeneration: number
}>
): string {
return `${providerGeneration}\0${source.relayPtyId ?? ''}\0${source.clientGeneration}\0${source.ownerGeneration}\0${source.deliveryToken}`
}
}