From 879f502e7f76fbc081f48b07b641bf561c7186c2 Mon Sep 17 00:00:00 2001 From: OrcaWin <293788423+OrcaWin@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:36:16 -0700 Subject: [PATCH] docs(ssh): reconcile final backpressure lifecycle --- config/reliability-gates.jsonc | 12 +- .../2026-07-27-ssh-relay-pty-backpressure.md | 573 +++++++++++++----- 2 files changed, 419 insertions(+), 166 deletions(-) diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index eb0c6b1a730..6c2d4a42f0f 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -3117,8 +3117,9 @@ "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-legacy-projection.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts --reporter=dot", - "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts --reporter=dot", "ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" ], @@ -3492,6 +3493,15 @@ "durationSeconds": 11.71, "summary": "Eleven deterministic source-intake, exit-deadline, remote-consumer, multiplex, writer, rollout, and deploy files passed 197 tests, including exit-time ordering, control-lane drain priority, and typed mismatched-principal recovery with retained authority isolation." }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-pty-source-restore-retry.test.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/providers/ssh-pty-notification-routing.test.ts src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts src/main/providers/ssh-pty-provider-exit-race.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 12.5, + "summary": "Eleven current-head deterministic files passed 134 tests across restore retirement, fail-closed recovery cancellation, partial ACK and token rotation, reclaimed-span replacement, mux lane fairness, provisional activation, and exit-proof ordering; no live topology was exercised." + }, { "date": "2026-07-28", "runner": "local", diff --git a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md index 3c38f97da81..8b0f6de94ac 100644 --- a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md +++ b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md @@ -2,9 +2,9 @@ Date: 2026-07-27 -Status: architecture and exact SSH V1 wire/API implemented behind an -experimental startup gate; current shared-worktree reconciliation complete; -exact-head review and validation pending +Status: architecture gate closed and exact SSH V1 wire/API implemented behind +an experimental startup gate; integrated lifecycle fixes reconciled at current +HEAD; live topology gaps remain explicit below Historical unbounded baseline: `badf91101babf96fa09cb79a8294f7e23b9f081c` (the implementation branch parent). The implementation was rebased onto @@ -46,41 +46,51 @@ implemented. In this PR: writer owns byte admission, priorities, `write(false)`, callbacks, and drain. - Spawn and attach responses carry immutable `sourceActivation` metadata. The main mux installs a provisional receive lease synchronously in - `beforeResolve`, and provider validation commits or rolls back that exact - lease before adjacent source notifications can escape. + `beforeResolve`; adjacent source notifications remain private to that lease + until provider validation commits, while rollback drops them and waits for + exact token-cancellation proof. - `SshRelaySession` owns grant/reconnect state, bounded recovery quarantine, exact recovery fencing, and cumulative ACK publication through the main SSH - mux. + mux. Failed recovery-cancellation publication or proof closes the exact + provider generation and its publishers once. - The main SSH intake owns one atomic admission across the model, desktop projection, and required remote consumers. Model completion and consumer terminality make ACKs eligible; only the mux write callback publishes them. - Desktop and remote replacement transactions carry immutable span identity. Reserve happens before publication, commit requires the exact - generation/sequence fence, and failure or stale publication rolls back - without credit. + generation/sequence fence, and failure, stale publication, or already + reclaimed spans roll back or prune idempotently without credit. +- Relay `restoreRequired` cancellation retains its exact delivery record until + the metadata response settles, then retires only that record so a retry can + mint a fresh token. Main exit timeout obtains cancellation proof before + transferring projections and committing reclamation, with generation checks + around every finalization step. +- The main-to-relay writer has three FIFO lanes: liveness, control, and + ordinary. Drain reselects liveness/control ahead of queued ordinary traffic, + while one ordinary frame is guaranteed after four control writes. The implementation is intentionally scoped: -| Surface | State in this PR | Evidence | -| ------------------------------------------- | ------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| Same-build direct SSH/deployed relay, V1 on | implemented; exact-head validation pending | deterministic relay/main contracts plus prior macOS-hosted Docker OpenSSH | -| Same-build SSH, V1 off | implemented bounded legacy mode | deterministic rollout and writer contracts | -| Reconnect/owner recovery | implemented | exact contiguous recovery, stale-owner one-shot retry, eight-wide reattach tests, Docker reconnect | -| Headed/headless remote consumers | source-range reserve/commit/rollback implemented | deterministic runtime/main seam only; no live paired-runtime claim | -| WSL stdio and Windows named pipe/ConPTY | shared transport paths preserved | deterministic/common-contract evidence only; no physical run | -| Local provider and local daemon | unchanged | outside SSH V1 negotiation; existing behavior only | -| Folder workspace | no `.git` dependency added | code-path review only; no dedicated live fixture | -| Prior-version/mixed-version peers | fail-closed or version-scoped legacy behavior preserved | deterministic rollout/deploy contracts; no live old binary | -| Ubuntu 20.04/glibc 2.31 | no native dependency added | cross-target relay build only; no physical packaging run | +| Surface | State in this PR | Evidence | +| ------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Same-build direct SSH/deployed relay, V1 on | implemented | current-head deterministic relay/main contracts plus prior macOS-hosted Docker OpenSSH | +| Same-build SSH, V1 off | implemented bounded legacy mode | deterministic rollout and writer contracts | +| Reconnect/owner recovery | implemented | exact contiguous recovery, fail-closed cancellation, eight-wide reattach tests, Docker reconnect | +| Headed/headless remote consumers | source-range rotation/replacement implemented | deterministic runtime/main seam only; no live paired-runtime claim | +| WSL stdio and Windows named pipe/ConPTY | shared transport paths preserved | deterministic/common-contract evidence only; no physical run | +| Local provider and local daemon | unchanged | outside SSH V1 negotiation; existing behavior only | +| Folder workspace | no `.git` dependency added | code-path review only; no dedicated live fixture | +| Prior-version/mixed-version peers | fail-closed or version-scoped legacy behavior preserved | deterministic rollout/deploy contracts; no live old binary | +| Ubuntu 20.04/glibc 2.31 | no native dependency added | cross-target relay build only; no physical packaging run | Docker SSH proves the Linux SSH provider and deployed-relay topology. It does not prove headed or headless paired-runtime behavior, WSL, Windows ConPTY or named pipes, local daemon/provider behavior, folder workspaces, prior-version processes, mixed-version clients, or the Ubuntu 20.04 packaging floor. -The prior executable evidence remains useful, but it does not validate the -current shared-worktree diff. Exact-head review, focused tests, formatting, -typecheck, and topology reruns remain pending unless explicitly listed below. +The prior Docker evidence remains useful but predates the final lifecycle +fixes. Current-head deterministic checks and document validation are listed +below; no new live topology proof is inferred from them. ## Verified baseline and migration boundary @@ -199,17 +209,19 @@ renderer-only cap. written again before drain. PTY admission stops before a reserved control/liveness capacity is consumed. 5. A spawn/attach token remains `activating` until its metadata-only response - crosses the sink fence, then remains `recovering` until all recovery - `pty.data` and the completion fence drain. No live `pty.data` is eligible - earlier. + crosses the relay sink fence. Main may receive source frames after its + synchronous provisional install, but they remain lease-private and cannot + project until provider/claim/exit validation commits that lease. Recovery + remains fenced until all recovery `pty.data` and completion drain. 6. Data is ordered within a token. `pty.exit` follows every accepted data frame and cannot bypass the sink gate or activation fence. Publishing exit seals new data but does not retire an uncredited suffix. 7. Every open token-owned span and consumer obligation ends exactly once in `settled`, `transferred`, or `canceled`; a transfer may pass through the non-terminal `transferring` state only while its named replacement fence is - outstanding. Records are never merely abandoned while their relay token can - remain live. + outstanding. A canceled `restoreRequired` delivery remains addressable until + its exact response settles and retires only if the map still names that + record. Records are never merely abandoned while their relay token can live. 8. Desktop, mobile, web, and agent-session consumers follow the explicit settlement policy below. A stalled recoverable desktop projection cannot freeze a healthy lossless remote view. @@ -223,13 +235,16 @@ renderer-only cap. replay tail and no live queue. 12. Decoder work, decoded bytes, model admission, and activation-hold turns are bounded. A self-imposed read or local-write pause rebases both liveness - clocks, and control/liveness service remains bounded under PTY and bulk - saturation. + clocks. Main-to-relay drain reselects liveness, then control, ahead of + ordinary PTY input; four consecutive control selections force one ordinary + selection when both lanes remain non-empty. 13. Disconnect, provider replacement, renderer reload, exit, disposal, and workspace removal have explicit bounded cleanup. After their required publication, transfer, cancellation, or generation-close proof, they leave no open span, token, writer callback, drain waiter, timer, cursor, or paused - PTY. + PTY. Recovery-cancellation publication failure or rejected proof fails + closed by closing only the owning provider generation and disposing its + mux, provider, publishers, provisional lease, and registration once. 14. All counters are finite safe integers. A transformed frame requires a valid `rawLength`; malformed, excessive, stale, and cross-client values cannot create credit or crash either process. @@ -251,7 +266,10 @@ renderer-only cap. 20. Desktop admission carries immutable span identity through model reserve, projection queue, renderer send, ACK, salvage, reload, and replacement. Failure either rolls back an uncommitted transaction or transfers a - committed obligation with proof. + committed obligation with proof. Exit timeout acquires cancellation proof + without mutation, transfers published projections, commits proof + reclamation, prepares/finalizes exit once, and rechecks the provider + generation before each post-proof mutation. 21. Projection drop, thinning, salvage, restore, and replay preserve current main's ordered terminal side-effect facts and bounded scanner state, including DEC mode 2031 subscribe/withdraw decisions. Pre-commit rollback @@ -272,6 +290,19 @@ clientGeneration, ownerGeneration, ptyIncarnation, deliveryToken)`. Each semantics. It may use the fixed-frame lane only from an empty, unsaturated producer epoch; ordinary PTY and reshapable bulk frames still obey the non-reserved capacity. +26. A lossless remote stream may advance its cumulative encoded-byte ACK to an + interior byte offset. Exact byte budget is released immediately, while the + covering immutable source frame remains unsettled until its complete + encoded boundary is crossed. +27. A lossless remote stream accepts an in-place source-token rotation only + when PTY ID, provider generation, and incarnation match; both client and + owner generations strictly increase; the token changes; and source/display + coordinates remain contiguous. Old or partially increasing identities are + rejected. +28. Snapshot replacement commits are idempotent and never synthesize byte or + source credit. Duplicate/closed ledger commits are no-ops; if cancellation + proof already reclaimed a reserved span, source replacement commit rejects, + rollback skips the absent span, and settle/detach prune cached mappings. ## Protocol @@ -281,8 +312,9 @@ Decision: `PtyConsumerSession` is a shared semantic state machine, not a universal transport protocol. The architecture gate is closed and is no longer waiting for exact wire/API names: the SSH adapter implements `sourceActivation`, source-bearing `pty.data`, `pty.recoveryComplete`, -cumulative `pty.ackData`, and token-scoped cancellation. Exact-head code -review and validation remain engineering gates. A relay-only +cumulative `pty.ackData`, and token-scoped cancellation. Current-head +deterministic validation is an engineering gate; unexecuted live topologies +remain separate promotion gates. A relay-only `pty.getCapabilities` followed by `pty.negotiateClient` remains rejected because it would create a second readiness authority beside the existing authenticated handshakes. @@ -310,6 +342,19 @@ additional interleavings: | Recovery-response cancellation authority ended at enqueue rather than writer settlement | The request remains cancellation-authoritative until its response settles; retry reuses the same provisional replacement identity instead of creating an unfenced owner. | | Snapshot replacement transferred every pending remote span at one sequence fence | Each immutable span retains its model-sequence end; replacement transfers only spans covered by the authoritative `SnapshotEnd`, while trailing spans remain live and replay in order. | +Current-head adversarial reconciliation closed these additional integrated +failures without widening the approved topology: + +| Finding | Implemented resolution | +| ---------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Invalid-checkpoint `restoreRequired` left a canceled delivery blocking retry | Keep the exact canceled record through response settlement, then delete it only if it is still current; retry mints a fresh activation token. | +| Recovery cancellation publication/proof rejection left ambiguous authority | Raise a typed recovery-cancellation failure and close the exact provider generation, disposing its mux, provider, publishers, lease, and registration exactly once. | +| Remote partial ACK or higher-generation token rotation detached the stream | Release exact cumulative byte credit at any valid interior offset without source settlement; admit only contiguous strictly higher client/owner generations with a new token. | +| Snapshot cleanup raced cancellation-proof reclamation | Make ledger replacement commit idempotent and make remote replacement commit reject reclaimed spans while rollback, settle, and detach prune them without dereference or new credit. | +| Drain resumed the main mux's old ordinary FIFO ahead of control | Select liveness, control, and ordinary from separate FIFO lanes; prioritize control after drain and force one ordinary write after four controls. | +| Same-turn source data escaped a provisional activation | Queue data privately under the provisional lease; commit publishes in order, while rollback drops it, restores only the active predecessor, and awaits exact cancellation proof. | +| Exit cancellation proof reclaimed spans before projection transfer | Acquire proof first, transfer projections, commit proof, prepare/finalize once, and fence every post-proof step against provider-generation close. | + The common state machine accepts only: - an authenticated principal and owner-eligibility decision from the adapter; @@ -703,12 +748,17 @@ The main mux supplies the receive fence. Spawn and attach requests register a synchronous `beforeResolve(result)` hook in their pending-request record. `handleResponse` parses the whole response and provisionally installs `sourceActivation` before resolving the Promise. The lease captures the exact -prior receive state and candidate token. Only after operation-ID result, -incarnation, claimed-owner, attach identity, stale-attempt, and -spawn/attach-exit validation succeeds does the provider commit it and update -`livePtyIds`. Any failure rolls back the exact provisional state and token -through the originating mux; it cannot erase a newer revision or cancel a -newer token. An identical already-installed activation is an idempotent no-op. +prior receive state, candidate token, and a private ordered list of same-turn +source frames. Only after operation-ID result, incarnation, claimed-owner, +attach identity, stale-attempt, and spawn/attach-exit validation succeeds does +the provider commit it, publish the held frames in order, and update +`livePtyIds`. Any failure first retires the candidate and drops its held data, +restores only the nearest non-retired predecessor, then waits for a valid +`pty.cancelDelivery` response from the originating mux. It cannot erase a +newer revision or cancel a newer token. An identical already-committed +activation is an idempotent no-op; an identical provisional activation is +stale. Fresh-spawn rollback that cannot prove cancellation surfaces +`execution_owner_unavailable` rather than claiming the old token retired. The receive cursor starts at `checkpointSourceEndSu`, not `recoveryEndSu`. Source-bearing notifications for the installed identity may @@ -718,10 +768,10 @@ fence commits. Unknown or mismatched source identities are rejected before `livePtyIds`, runtime, renderer, or source obligations change. Output produced while a token is activating is retained by its bounded shared -cursor and can pause the native PTY; it is never sent early. Main drops -unknown-token notifications before `livePtyIds.add` or any other side effect -and records a protocol violation. No unknown-token hold is needed because the -wire fence is authoritative. +cursor and can pause the native PTY; it is never sent early. Output arriving +after main's synchronous install but before authority commit is bounded by the +provisional lease and is not visible to runtime, desktop, remote consumers, or +`livePtyIds`. Main drops unknown-token notifications before any side effect. Before issuing attach or any same-client replacement, main serializes that `(providerGeneration, ptyIncarnation)` through a migration fence: freeze new @@ -740,9 +790,13 @@ proved fence it sends `{ status: 'checkpointUnavailable' }`. The relay accepts the checkpoint only for the same incarnation and within `[oldCreditedEndSu, retainedLiveEndSu]`. It never clamps a value forward, trusts a value beyond sent/retained data, or replays freed pre-credit source; -an invalid or uncovered checkpoint returns the token-free `restoreRequired` -arm. It does not activate a subscription or stream a partial gap; main either -restores an authoritative model generation before retry or surfaces the gap. +an invalid or uncovered checkpoint cancels the attempted delivery and returns +the token-free `restoreRequired` arm. The canceled record remains installed and +cancellation-authoritative until that metadata response settles. Settlement +deletes it only if the delivery map still points to the exact record and wakes +capacity; a retry can then mint a fresh token. It does not activate a +subscription or stream a partial gap; main either restores an authoritative +model generation before retry or surfaces the gap. Recovery bodies use the ordinary source-bearing `pty.data` schema. The new token initializes `sentEndSu = creditedEndSu = checkpointSourceEndSu`, and @@ -793,11 +847,17 @@ the model nor desktop projection. A failed or stale recovery attempt cancels only its own replacement token. It never sends cancellation on a newer mux, clears a newer checkpoint, shuts down the physical PTY/process, deletes PTY ownership, or expires the owner -lease. A valid token-scoped cancellation proof is applied locally when it -matches; otherwise the lease remains detached and retryable. A fresh relay -that no longer retains the cached owner returns typed error `-32041`; main -clears only the cached owner/checkpoints and retries `pty.openClient` exactly -once without `resume`. +lease. A valid matching token-scoped cancellation proof is applied locally and +leaves the lease detached and retryable. If the current attempt cannot publish +cancellation or local proof application rejects, the session fails closed: +it closes the exact provider generation with +`ssh_source_recovery_cancellation_failed`, disposes that generation's mux, +provider, ACK/cancellation publishers, provisional lease, and registration +once, and reconnects without clearing physical PTY ownership. If the attempt +is already stale, it performs no cleanup against the replacement. A fresh +relay that no longer retains the cached owner returns typed error `-32041`; +main clears only the cached owner/checkpoints and retries `pty.openClient` +exactly once without `resume`. Idempotent agent-session spawn has two layers. The cached `agentSessionCreateOperationId` promise returns only the physical PTY @@ -820,11 +880,15 @@ survives; the next retry can subscribe to the retained PTY. Split policy from lifecycle and transport: -- `pty-output-span-ledger.ts` owns immutable source spans, token cursors, - cumulative validation, reconnect-grace ownership, and reclamation. -- `pty-output-scheduler.ts` owns DRR selection, 16 Ki-su publication, - per-client cursors, recovery, exit barriers, budgets, and additional- - subscriber eviction. +- `pty-source-credit-ledger.ts`, `pty-source-credit-record.ts`, and + `pty-source-replay-index.ts` own cumulative token state, immutable retained + source records, exact replay ranges, rotation, and reclamation. +- `relay-pty-source-publication.ts`, + `relay-pty-source-activation.ts`, + `relay-pty-source-send-scheduler.ts`, and + `relay-pty-source-exit-publication.ts` own activation/restore settlement, + 16 Ki-su publication, recovery, exit barriers, budgets, and subscriber + scheduling. - `dispatcher-client-writer.ts` owns the one writer per sink, activation fences, lane queues, drain callbacks, and close settlement. @@ -878,18 +942,18 @@ and request model restore; it may not over-credit, truncate, or send an oversized or permanently inadmissible frame. The existing 100 Ki-su replay-tail policy and the chunked -`RecentPtyOutputBuffer` representation from `79ec57d04` remain. Slice 5c adds a -bounded parallel source-range index or extends that buffer with indexed -records; it must preserve the current append-path performance and equivalence -tests and must not restore the former rolling-string re-slice. Legacy replay -still materializes the same display tail. V1 restore additionally preserves +`RecentPtyOutputBuffer` representation from `79ec57d04` remain. +`PtySourceReplayIndex` now provides the bounded parallel source-range index +without restoring the former rolling-string re-slice. Legacy replay still +materializes the same display tail. V1 restore additionally preserves incarnation, exact source ranges, and transform metadata. Index eviction -follows the buffer's exact retained prefix and never fabricates a mapping. +follows the buffer's exact retained prefix and never fabricates a mapping; the +existing append-path equivalence/performance contracts remain in force. ### Scheduling and fairness -Use deficit round-robin over PTYs, then rotate subscribed clients for the -selected PTY. One scheduling turn admits at most: +The relay scheduler uses deficit round-robin over PTYs, then rotates subscribed +clients for the selected PTY. One scheduling turn admits at most: - two PTY frames; - 32 Ki su; @@ -995,13 +1059,21 @@ remote mapping, and ACK publication record. In order, it: The barrier has the same charged 30-second deadline. At expiry, main first sends token-scoped `pty.cancelDelivery` and waits up to 10 seconds for its -proof. That proof permits failing/transferring remote streams, resetting only -this PTY's model generation with `restoreRequired`, and reporting incomplete -exit instead of successful lossless completion. Only if the cancellation -request or proof cannot publish does main close the provider so generation -cleanup proves cancellation; unrelated PTYs are not torn down on the ordinary -timeout path. A late emulator callback cannot mutate the reset generation. -Thus relay sink ordering and main asynchronous model ordering are both proved. +proof without applying it. If the exact barrier/generation still owns the +exit, main transfers published desktop projections to +`ssh-exit-delivery-canceled`, commits the proof to reclaim source spans, +prepares exit once, rechecks generation ownership, finalizes exit, rechecks +again, then closes the projection. This order prevents proof commit from +reclaiming a span before its published projection transfers. + +If provider-generation close wins while proof is pending, it rejects and +removes the barrier plus its cancellation/preparation ownership; the late +proof performs no transfer, commit, preparation, final exit, or projection +close. If cancellation publication or proof fails while the barrier remains +current, main closes the provider so generation cleanup proves cancellation; +unrelated PTYs are not torn down on the ordinary successful timeout path. A +late emulator callback cannot mutate the reset generation. Thus relay sink +ordering and main asynchronous model ordering are both proved. ## Dispatcher and transport drain @@ -1241,8 +1313,9 @@ value, so retries cannot lose an eligible prefix. ### Bounded asynchronous model admission -Replace the unmetered headless-emulator `writeChain` contract with a charged -FIFO per PTY and global scheduler. `acceptPtyData` returns a Promise receipt: +The charged FIFO per PTY and global scheduler replace the unmetered +headless-emulator `writeChain` contract. `acceptPtyData` returns a Promise +receipt: 1. before capturing the frame, reserve its charged retained bytes against the per-PTY and global model-admission budgets; @@ -1357,14 +1430,23 @@ ordered composite list rather than collapsing them into a proportional range. An ACK must be a finite safe integer in `previousAckedEndByte <= ackedEndByte <= lastAcceptedByte`; excessive values are rejected and counted, never clamped into full settlement. An in-range -partial-frame ACK is recorded but settles no source units; only complete -recorded frame boundaries settle their exact ordered source ranges. UTF-8 -width, JSON escaping, `rawLength`, and display transforms are never converted -proportionally. -Before stream replacement, snapshot recovery, or detach, create the -replacement owner and atomically transfer every remaining mapping; otherwise -cancel that consumer and its bounded stream. Old stream generations cannot -settle new mappings. +partial-frame ACK advances the byte watermark and releases exactly that byte +budget, but retains the covering frame's immutable source mapping and settles +no source units. Only complete recorded frame boundaries settle their exact +ordered source ranges. UTF-8 width, JSON escaping, `rawLength`, and display +transforms are never converted proportionally. + +The same attached stream may cross a source-token rotation without transferring +its older unsettled frame. Admission requires the same PTY ID, provider +generation, and incarnation; strictly greater client and owner generations; a +different delivery token; and exact source/display continuity. The ledger then +rebinds future frames to the new identity while older frames retain their own +immutable identity and settle at their encoded boundaries. Equal, lower, +partially advanced, cross-provider, cross-incarnation, or same-token rotations +are rejected. Before stream replacement, snapshot recovery, or detach, create +the replacement owner and atomically transfer every remaining mapping; +otherwise cancel that consumer and its bounded stream. Old stream generations +cannot settle new mappings. The main registry stores the immutable source identity with each admitted remote mapping, not only its `spanId`. A replacement reservation moves the @@ -1373,9 +1455,10 @@ headless `SnapshotEnd` publication at or beyond the reservation's required sequence commits them. Stale generation, failed publication, disconnect, or explicit rollback restores the original live-stream obligation. If token-scoped cancellation proof has already reclaimed the authoritative span, -late ACK, detach, rollback, and replacement commit prune the cached mapping -and become idempotent no-ops; they never dereference reclaimed state or create -credit. +late ACK/settle prunes the cached mapping, detach skips absent spans, and a +replacement commit returns false without dereferencing the reclaimed record. +Rollback then deletes the exact reservation and skips already-absent spans. +These paths are idempotent and create no credit. ACK-pending output overflow uses the same transaction. For a source-mapped stream, serialize the bounded snapshot, reserve only immutable spans whose @@ -1389,6 +1472,10 @@ publication, or commit failure calls `rollbackRemoteTerminalSourceRangeReplacement`, preserves the live obligations, reports the stream error, and detaches that stream. One `ackRecoverySnapshotInFlight` flag bounds the operation to single-flight. +At the encoded-byte ledger, snapshot replacement commit is itself idempotent: +duplicate commits and commits after close are no-ops, preserve cumulative byte +coordinates, and remove only source mappings covered by the authoritative +snapshot sequence. The stall policy is deliberately split. Desktop-only parse failure reaches its bounded projection cap, transfers to model restore, and lets upstream credit @@ -1415,25 +1502,32 @@ explicit/connection-close proof. The transaction creates the replacement owner before marking source `transferring`; only exact recovery-complete coverage terminally transfers it. -| Event | Mandatory transition | -| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| spawn/attach response decoded | install provisional `sourceActivation` in synchronous `beforeResolve`; commit after all provider/exit validation, otherwise restore exact prior state and cancel only candidate token | -| model accepts; desktop/mobile parses | settle that consumer | -| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker | -| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting | -| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains | -| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof | -| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore | -| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage | -| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range | -| empty or non-empty recovery completion | require checkpoint-to-`recoveryEndSu` continuity, retain that live-start anchor after activation, then admit only an exactly contiguous live frame | -| recovery completion lacks writer capacity | retain range/fence state and retry on capacity; keep at most one admitted completion attempt | -| stale/overlapping recovery attempt | cancel only that attempt's token; never mutate the replacement mux, checkpoint, PTY ownership, physical process, or owner lease | -| remote snapshot replacement | reserve immutable span IDs/ranges, commit only after current-generation `SnapshotEnd` coverage, otherwise roll back | -| remote ACK-pending overflow snapshot | reserve covered spans, publish, commit exact snapshot identity/sequence, then trim; rollback and detach on failure | -| late remote detach after token cancellation | prune already-reclaimed mappings and detach idempotently without dereference or new credit | -| explicit live-token reset | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport | -| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof | +| Event | Mandatory transition | +| ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| spawn/attach response decoded | install provisional `sourceActivation` in synchronous `beforeResolve`; hold same-turn data privately, commit/publish after validation, or drop/restore and await exact cancellation proof | +| invalid-checkpoint `restoreRequired` | keep the canceled relay record through response settlement; retire only the still-current record, wake capacity, and permit a fresh-token retry | +| model accepts; desktop/mobile parses | settle that consumer | +| hidden thinning, empty transform, pending-cap salvage | preserve ordered terminal facts/scanner state, then transfer desktop to model and emit restore marker | +| renderer reload/destroy/send failure/heal | transfer all desktop obligations to model before clearing queue/accounting | +| pane closes while provider/token remains live | transfer recoverable views, then request token cancel if no required consumer remains | +| PTY exit | relay seals token; main retains prior receipts/projections/remotes; runtime/renderer exit only after terminality or bounded cancellation proof | +| provider replacement/reconnect | close old client generation; transfer only exact ranges proven by replacement recovery, otherwise cancel and restore | +| relay `pty.deliveryCanceled` | without replacement, cancel matching remainder and restore; with replacement, enter `transferring` pending coverage | +| same-client token supersession | create replacement first, transfer exact covered remainder after recovery, cancel any uncovered range | +| empty or non-empty recovery completion | require checkpoint-to-`recoveryEndSu` continuity, retain that live-start anchor after activation, then admit only an exactly contiguous live frame | +| recovery completion lacks writer capacity | retain range/fence state and retry on capacity; keep at most one admitted completion attempt | +| stale/overlapping recovery attempt | cancel only that attempt's token; never mutate the replacement mux, checkpoint, PTY ownership, physical process, or owner lease | +| current recovery cancellation publication/proof fails | close the exact provider generation and release its mux, provider, publishers, provisional lease, and registration once; retain physical PTY ownership for reconnect | +| partial remote cumulative byte ACK | advance/release exact byte credit; retain the covering source frame and obligation until a complete recorded boundary | +| contiguous higher-generation source token | keep the stream attached, bind later frames to the new identity, and preserve unsettled older-token frame mappings | +| remote snapshot replacement | reserve immutable span IDs/ranges, commit only after current-generation `SnapshotEnd` coverage, otherwise roll back | +| snapshot commit after proof reclaimed a span | reject commit without dereference; rollback removes the exact reservation and skips absent spans; later settle/detach prunes cached mappings | +| remote ACK-pending overflow snapshot | reserve covered spans, publish, commit exact snapshot identity/sequence, then trim; rollback and detach on failure | +| late remote detach after token cancellation | prune already-reclaimed mappings and detach idempotently without dereference or new credit | +| exit timeout receives cancellation proof | while generation-current, transfer projections, commit proof, prepare/finalize once, and close projection with generation checks between mutations | +| provider generation closes during exit proof | reject/remove barrier ownership; a late proof cannot commit, prepare, finalize, or publish exit | +| explicit live-token reset | `pty.cancelDelivery` response proves relay cancellation before local discard; failure closes the provider transport | +| relay/client dispose | relay cancels token/cursors; main cancels only after close-generation proof | There is no “abandon live token” transition. A main-side path that cannot prove settlement, transfer, or relay cancellation must keep the ledger or close the @@ -1463,6 +1557,9 @@ relay token: recovering --completion admitted--> completion-in-flight(single) completion-in-flight --write success--> active completion-in-flight --write error--> recovering(exact replacement/cleanup) + recovering --invalid checkpoint--> restore-canceled-awaiting-response + restore-canceled-awaiting-response --exact response settles/current record--> + closed(delete record, wake capacity) activating --stale/close--> closed(cancel) activating|recovering|active --same-client replacement--> closing(superseded) active --native exit--> sealed-unsettled @@ -1477,6 +1574,13 @@ writer: writable --write(false)--> saturated --drain--> writable writable|saturated --close/error/detach--> closed +main mux writer selection: + any queued liveness --> select liveness + control queued and (ordinary empty or controlsSinceOrdinary < 4) --> select control + ordinary queued --> select ordinary(reset controlsSinceOrdinary) + control queued after ordinary empty --> select control + drain --> writable(re-run the same selection; preserve FIFO inside each lane) + main obligation: open --consumer receipt--> settled open --replacement created atomically--> transferring @@ -1497,9 +1601,14 @@ main token span: --cancel response/client-generation close--> canceled main receive token: - unseen --beforeResolve--> provisional(cursor = checkpointSourceEndSu) - provisional --all spawn/attach/exit validation--> receiving-activation(commit lease) - provisional --validation failure--> prior exact state(rollback candidate token) + unseen --beforeResolve--> provisional(cursor = checkpointSourceEndSu, heldData = []) + provisional --same-turn contiguous data--> provisional(append heldData; no projection) + provisional --all spawn/attach/exit validation--> + receiving-activation(commit lease, publish heldData in order) + provisional --validation failure--> + rollback(drop heldData, restore active predecessor, request exact cancellation) + rollback --valid cancellation response--> prior exact state + rollback --publication/proof rejects--> fail operation(execution owner unavailable) receiving-activation --validated response/no recovery--> active receiving-activation --quarantine exact checkpoint..recoveryEnd + complete fence--> active(expectedLiveStart = recoveryEnd) @@ -1509,15 +1618,28 @@ main receive token: exit-sealed --prior model/projection/remote terminal--> exit-ack-queued exit-ack-queued --runtime + renderer exit cleanup--> exited-awaiting-ack-publication exited-awaiting-ack-publication --ACK success/generation close--> closed - exit-sealed --deadline--> canceling-exited-token - canceling-exited-token --token cancellation proof--> closed(restore-required) - canceling-exited-token --proof cannot publish--> closing-provider + exit-sealed --deadline--> acquiring-exit-cancellation-proof + acquiring-exit-cancellation-proof --proof + generation current--> + transfer-projection -> commit-proof -> prepare-once -> finalize -> close-projection + acquiring-exit-cancellation-proof --generation closes--> + closed(reject barrier; late proof has no authority) + acquiring-exit-cancellation-proof --proof cannot publish while current--> closing-provider receiving-activation|active --overflow/cancel/close--> closed +remote source ledger: + frame-open --partial cumulative byte ACK--> + frame-open(release exact byte delta; retain source mapping) + frame-open --ACK reaches encoded frame end--> frame-settled(settle exact source ranges) + old-token frame-open --contiguous strictly higher client+owner generation/new token--> + old-token frame-open + new-token receiving + receiving --stale/partial generation advance or identity discontinuity--> reject + remote projection replacement: live --reserve immutable spans + required SnapshotEnd seq--> transferring transferring --current generation SnapshotEnd covers required seq--> transferred transferring --stale/failure/disconnect--> live(rollback) + transferring --authoritative span already reclaimed--> + commit-rejected -> rollback(skip absent span, delete reservation) live|transferring --authoritative token already canceled--> detached(prune cached spans) remote ACK-overflow recovery: @@ -1639,14 +1761,44 @@ function finishRecovery(candidate: RecoveryCandidate, fence: RecoveryComplete): installLiveStartAnchor(candidate.identity, fence.recoveryEndSu) } +async function rollbackProvisionalActivation(lease: ReceiveLease): Promise { + lease.dropHeldDataAndRestoreActivePredecessor() + const proved = await cancelExactTokenThroughOriginatingMux(lease.identity) + if (!proved) throw executionOwnerUnavailable() +} + +function acknowledgeRemoteBytes(stream: RemoteStream, ackedEndByte: number): void { + const delta = stream.validateAndAdvanceCumulativeByteAck(ackedEndByte) + stream.byteBudget.release(delta) + for (const frame of stream.completeFramesThrough(ackedEndByte)) { + settleExactSourceRanges(frame.sourceRanges) + } +} + function replaceRemoteConsumer(stream: RemoteStream, requiredSeq: number): void { const reservation = remoteRanges.reserveImmutableSpans(stream, requiredSeq) publishSnapshotEnd(stream).then( - (publication) => remoteRanges.commitIfCurrentAndCovered(reservation, publication), + (publication) => { + if (!remoteRanges.commitIfCurrentAndCovered(reservation, publication)) { + remoteRanges.rollbackSkippingReclaimed(reservation) + } + }, (error) => remoteRanges.rollback(reservation, error) ) } +async function cancelTimedOutExit(exit: ExitBarrier): Promise { + const proof = await requestCancellationProofWithoutMutation(exit.identity) + if (!exit.isCurrentGeneration()) return + transferPublishedProjection(exit.identity) + commitCancellationProof(proof) + prepareExitOnce(exit.identity) + if (!exit.isCurrentGeneration()) return + finalizeExit(exit.identity) + if (!exit.isCurrentGeneration()) return + closeProjection(exit.identity) +} + async function recoverAckOverflow(stream: RemoteStream): Promise { const snapshot = await serializeBudgetedSnapshot(stream) const reservation = reserveCoveredRemoteSpans(stream, snapshot.seq) @@ -1671,7 +1823,7 @@ the range queue and is never interpreted directly as source units. ## Incremental frame decoding -Extend the actual main transport contract: +The actual main transport contract is extended as follows: ```ts type MultiplexerTransport = { @@ -1696,15 +1848,25 @@ Readable contract. WSL child stdin/stdout exposes the same metrics, callback/drain, and pause/resume adapter. The startup residue is delivered once before later data. -`SshChannelMultiplexer` also owns one main-to-relay writer. Its lanes are -liveness, interactive/control (requests, PTY input, cancellation, coalesced -PTY ACK), and producer-owned bulk. It applies the same effective reserve and -applies the same one-keepalive saturated-epoch exemption; all ordinary writes -wait for their per-write callback and `onWriteDrain` after saturation. -Reshapable bulk is sliced to non-reserved capacity; fixed filesystem frames -use their exclusive empty-sink admission. Priority is rechecked between -frames. Thus an ACK burst or file/Git stream cannot build an invisible queue -ahead of keystrokes, and V1 ACK settlement has a concrete write fence. +`SshChannelMultiplexer` also owns one main-to-relay writer with exactly three +bounded FIFO lanes: + +1. `liveness` for the coalesced probe, including the one saturated-epoch + bypass; +2. `control` for source ACK, cancellation, exit, RPC request, and response + frames; +3. `ordinary` for `pty.data`, including PTY input/paste traffic. + +The scheduler rechecks priority after every frame and whenever drain reopens +the writer. Liveness wins first. Control wins over queued ordinary traffic +until four control frames have been selected while ordinary remains queued; +the next selection must be one ordinary frame and resets the counter. FIFO is +preserved within each lane. All non-bypass writes still wait for their +per-write callback and `onWriteDrain` after saturation. Producer-side bulk +shaping and fixed filesystem-frame admission remain with their existing +owners; they do not create a fourth mux-writer lane. Thus control cannot remain +behind an old ordinary backlog after drain, ordinary input cannot starve under +a sustained control stream, and V1 ACK settlement has a concrete write fence. On the relay, the initial client uses `process.stdin.pause()/resume()` and each accepted Unix-socket/named-pipe client uses `socket.pause()/resume()`. @@ -1763,9 +1925,16 @@ invalidates the lease so a later election starts a new generation. Each transition: - cancel read-pause epochs, writer callbacks, and scheduled work; -- reject queued control frames; +- reject queued liveness, control, and ordinary mux frames, clear lane heads, + and reset the four-control fairness counter; - roll back uncommitted receive-activation leases to their exact prior - revision and cancel only the provisional token through its originating mux; + active predecessor, drop lease-private data, and await cancellation of only + the provisional token through its originating mux; +- fail closed on the exact provider generation when a current recovery cannot + publish cancellation or validate its proof; provisional spawn rollback + failure instead surfaces execution ownership as unavailable; +- retain canceled `restoreRequired` relay deliveries until their response + settlement callback, then delete only the exact still-current record; - clear or transfer one recovery-completion attempt and its capacity listener; - move active PTY cursors to reconnect grace or cancel them; - preserve sealed-unsettled suffixes until the relay applies their ACK, exact @@ -1778,6 +1947,10 @@ transition: - cancel uncommitted terminal-fact publications, restore their prior scanner snapshots, and transfer committed projection scanner state exactly once; - roll back any remote snapshot replacement before clearing ACK-overflow state; +- preserve partial remote byte watermarks while retaining covering source + frames, and prune cached mappings whose authoritative spans were reclaimed; +- reject and remove exit barriers plus cancellation/preparation ownership on + provider-generation close so late proofs cannot publish final exit; - release shared spans no longer referenced; - recompute native PTY pause state; - expire pending RPC ownership. @@ -1819,11 +1992,13 @@ Recovery receipt is transactional. Main retains quarantined recovery `pty.data`, the candidate mux, token, checkpoint, and `recoveryEndSu` until the exact completion fence validates. It commits no partial body to the model. On failure it requests cancellation only through the candidate mux and accepts -only matching proof; a stale attempt may not touch current mux state. After a -successful empty recovery it keeps `recoveryEndSu` as a live continuity anchor -until the first live frame arrives. Failure marks the existing PTY lease -detached and retryable rather than deleting ownership or terminating the -physical process. +only matching proof; a stale attempt may not touch current mux state. A +current attempt whose cancellation publication or proof rejects closes the +exact provider generation and its registered resources once, rather than +leaving ambiguous authority. After a successful empty recovery it keeps +`recoveryEndSu` as a live continuity anchor until the first live frame arrives. +A proved recovery failure marks the existing PTY lease detached and retryable +rather than deleting ownership or terminating the physical process. For an operation-ID spawn replay, physical result lookup completes first; the current request then creates and fences a new subscription as specified above. @@ -2030,8 +2205,8 @@ boundaries even though they ship in one PR. legacy PTY frame. - Preserve the current 8 ms/1 ms PTY cadence and current replay semantics. The replay tail stays the chunked `RecentPtyOutputBuffer` added in - `79ec57d04`; a later source ledger must wrap or index it rather than replace - it with the former per-append rolling string. + `79ec57d04`; `PtySourceReplayIndex` now indexes it without replacing it with + the former per-append rolling string. - This slice bounds relay writable queues for legacy main, WSL, direct SSH, and detached relay socket/named-pipe clients without changing the wire. The separate local terminal daemon is not exercised or changed. @@ -2084,16 +2259,16 @@ boundaries even though they ship in one PR. ### Slice 5c: exit, reconnect, and replay — implemented -- Add sealed-unsettled exit, cancellation proofs, owner reconnect grace, +- Implements sealed-unsettled exit, cancellation proofs, owner reconnect grace, token replacement, and exact transfer. -- Carry recovery as contiguous source-bearing `pty.data` from checkpoint to +- Carries recovery as contiguous source-bearing `pty.data` from checkpoint to recovery end; fence live output with single-flight, capacity-retryable `pty.recoveryComplete` writer settlement. -- Add a bounded source-range index beside the current +- Adds a bounded source-range index beside the current `RecentPtyOutputBuffer`, preserving its append performance and legacy output. -- Do not enable reconnect V1 until late-ACK, timeout, supersession, and - generation-close oracles pass. +- Reconnect V1 remains under the startup gate; its late-ACK, timeout, + supersession, and generation-close deterministic oracles pass. ### Slice 5d: required remote consumers — deterministic seam implemented @@ -2115,7 +2290,28 @@ boundaries even though they ship in one PR. SSH. - Keep best-effort remote streams outside upstream obligations. -### Slice 5e: remaining topology validation and adapters — deliberately deferred +### Slice 5e: adversarial lifecycle reconciliation — implemented + +- Retain canceled invalid-checkpoint deliveries through exact + `restoreRequired` response settlement, then retire only the current record. +- Fail closed on the exact provider generation when a current recovery + cancellation cannot publish or its proof rejects; release generation-owned + resources once without clearing physical PTY ownership. +- Allow partial cumulative remote byte ACKs to release byte budget without + settling an incomplete source frame, and admit only contiguous strictly + higher client/owner generations with a new token. +- Make encoded snapshot replacement commit idempotent; reject reclaimed-span + commits and make rollback, settle, and detach prune absent spans safely. +- Split main-to-relay writes into liveness/control/ordinary FIFO lanes, select + control first after drain, and guarantee ordinary progress after four + controls. +- Hold provisional activation data until provider/claim/exit validation + commits; rollback drops it, restores only the active predecessor, and awaits + exact cancellation proof. +- Acquire exit cancellation proof before projection transfer and proof commit; + prepare/finalize once and fence every post-proof step by provider generation. + +### Slice 5f: remaining topology validation and adapters — deliberately deferred - Local in-process and daemon hello remain unchanged; neither adopts SSH framing or source credit in this PR. @@ -2142,6 +2338,10 @@ recorded evidence include: - main model admission, desktop identity/rollback, exit barriers, remote reserve/commit/rollback, token cancellation, exact recovery continuity, eight-wide reconnect, retention budgets, rollout latching, and deploy policy; +- current-head restore-response retirement, fail-closed recovery cancellation, + partial remote ACK, higher-generation token rotation, reclaimed-span + idempotence, three-lane mux fairness, provisional claim gating, and + generation-fenced exit proof ordering; - the four-test Docker OpenSSH/deployed-relay suite for source-window plateau, concurrent typing, fixed-size filesystem/Git churn, and owner reconnect. @@ -2166,16 +2366,36 @@ churn was 148.1/161.1 ms with 93 bulk reads; owner reconnect completed in 15.7 seconds. These measurements are direct SSH/deployed Linux relay evidence only. -Those results predate the current activation, recovery-completion, early-ACK, -subscriber-isolation, ACK-overflow snapshot, and split-SFTP credential diff. -Their exact-head focused and broader reruns remain pending. +The Docker results predate the final adversarial lifecycle reconciliation and +are not current-head proof for those races. The current-head deterministic +commands below cover the integrated fixes; a direct Docker rerun and all other +live topologies remain separate. + +At current HEAD, this focused command passed 11 files and 134 tests: + +```bash +pnpm exec vitest run --config config/vitest.config.ts \ + src/relay/relay-pty-source-restore-retry.test.ts \ + src/main/ssh/ssh-relay-session-recovery-races.test.ts \ + src/main/runtime/rpc/terminal-source-range-ledger.test.ts \ + src/main/runtime/rpc/terminal-multiplex.test.ts \ + src/main/ssh/ssh-multiplexer-transport-writer.test.ts \ + src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts \ + src/main/providers/ssh-pty-notification-routing.test.ts \ + src/main/providers/ssh-pty-provider-agent-session-create-operation.test.ts \ + src/main/providers/ssh-pty-provider-exit-race.test.ts \ + src/main/ipc/ssh-pty-output-exit-deadline.test.ts \ + src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts --reporter=dot +``` The deterministic recovery seam specifically covers an empty recovery followed by a gapped live frame, overlapping reconnect attempts, late frames after -cancel, typed stale-owner retry, and the negative assertions that no physical -PTY shutdown, provider-state clear, ownership delete, or lease expiry occurs. -The remote-consumer seam covers token cancellation followed by late detach and -replacement commit, proving reclaimed span IDs are pruned idempotently. +cancel, typed stale-owner retry, response-settled `restoreRequired` retirement, +and fail-closed exact-generation cleanup. It also asserts that a proved +token-local failure causes no physical PTY shutdown, ownership delete, or +lease expiry. The remote-consumer seam covers partial cumulative ACK, safe +higher-generation token rotation, and token cancellation followed by late +detach/replacement commit, proving reclaimed span IDs are pruned idempotently. ### Normative unit and property matrix @@ -2196,9 +2416,10 @@ replacement commit, proving reclaimed span IDs are pruned idempotently. controls, and transformed spans where `rawLength !== data.length`; assert the budget arithmetic and reject transformed frames without valid `rawLength`; - run the shared semantic state-machine suite and the implemented SSH - socket/named-pipe adapter suite; keep local, daemon, direct-stdio/WSL, and - remote-runtime adapters as distinct unimplemented or unexecuted topology - claims rather than inferring equivalence; + socket/named-pipe adapter suite; local and daemon V1 adapters remain + unimplemented, direct-stdio/WSL has deterministic common-transport evidence + only, and remote-runtime source mapping is implemented at a deterministic + seam but unexecuted in a live paired topology; - verify SSH session-owner election with V1 requested and with session-granted same-build legacy, invalid/stale credential or lease rejection, 30-second lease expiry, and atomic reconnect generation transfer through POSIX sockets @@ -2213,24 +2434,34 @@ replacement commit, proving reclaimed span IDs are pruned idempotently. once, report its exact remaining span, and transfer only matching recovery coverage while canceling the already-checkpointed prefix without re-ingest, duplicate output, or an open cursor; +- keep one lossless remote stream attached across an unsettled old-token frame + and a contiguous new-token frame; require strictly higher client and owner + generations, the same provider/incarnation, a new token, and reject every + stale or partially advanced identity; - verify metadata response callbacks precede source-ranged recovery and live data in V1, including saturation with older control; use multi-frame all-control-character gaps to prove source bodies remain ordinary producer-owned `pty.data` rather than control responses; - feed response, recovery, completion fence, and first live data in one decoder turn; prove synchronous `beforeResolve` installs a provisional cursor at - `checkpointSourceEndSu`, all provider/claim/exit validation precedes lease - commit, rollback restores the exact prior state/token, and exact gap spans - reach the model once through `recoveryEndSu`; + `checkpointSourceEndSu`, same-turn data remains unprojected until all + provider/claim/exit validation commits the lease, rollback drops held data, + restores only the active predecessor, awaits exact token-cancellation proof, + and exact gap spans reach the model once through `recoveryEndSu`; - reject wrong-incarnation checkpoints and checkpoints below old credit or beyond retained live end; require a token-free `restoreRequired` response - with no partial recovery or forward clamping; + with no partial recovery or forward clamping; retain the canceled record + through response settlement, retire only the exact current record, then + prove retry mints a fresh token and publishes one live source frame; - send identical recovery `pty.data` for distinct tokens within one second and prove token/range identity, not a wall-clock fingerprint, controls admission; - retry an operation-ID spawn after commit/before response from a new client; prove it receives a fresh subscription and the stale client cannot ACK; - assert `write(false)` admits exactly one frame, preserves the control reserve, and admits no ordinary later frame before drain; +- queue ordinary PTY input before source ACK, cancellation, exit, request, and + response frames; after drain require liveness/control selection first, FIFO + within all three lanes, and one ordinary selection after four controls; - deliver a cumulative ACK reentrantly before the source writer callback; reserve only the exact same-token pending boundary, then prove successful settlement applies it and failed settlement permits only an exact retry; @@ -2268,6 +2499,10 @@ replacement commit, proving reclaimed span IDs are pruned idempotently. migration fence either checkpoints after the final guarded receipt or times out to a model-generation reset/restore, with no late mutation or duplicate gap ingestion; +- reject recovery cancellation publication and local proof application while + the attempt is current; prove the exact provider generation, mux, provider, + publishers, provisional lease, and registration close once, while unrelated + generations and physical PTY ownership remain untouched; - prove renderer ACK/heal/write-off on V1 settles/transfers via display ranges and emits no legacy `acknowledgeDataEvent` wire delta; - admit interleaved spans from two token generations and prove every desktop @@ -2278,12 +2513,15 @@ replacement commit, proving reclaimed span IDs are pruned idempotently. - generate remote UTF-8, JSON-escaped, transformed, split, and composite frames; keep legacy `ackOutput: 1` delta ACKs outside source obligations; negotiate `ackOutputSourceRanges: 1`, then apply partial, excessive, and stale-generation - cumulative ACKs and prove excessive rejection plus source settlement only at - recorded frame boundaries, with send-stop without auto-transfer at the cap - and atomic mapping transfer on an explicit reconnect/detach; + cumulative ACKs and prove a partial ACK releases only its exact byte delta, + retains the covering source frame, and settles source only at recorded frame + boundaries, with send-stop without auto-transfer at the cap and atomic + mapping transfer on an explicit reconnect/detach; - overflow ACK-pending remote output and inject snapshot serialization, publication, exact-generation commit, and rollback failures; prove covered - chunks trim only after commit and recovery remains single-flight; + chunks trim only after commit and recovery remains single-flight; after + cancellation proof reclaims a reserved span, require commit rejection, + idempotent rollback, and safe late settle/detach pruning; - feed current mobile notification epochs and watermarks into terminal ACK handlers and prove they are rejected without changing any source obligation; - extend `terminal-output-frame-chunks-equivalence.test.ts` so composite source @@ -2311,9 +2549,11 @@ replacement commit, proving reclaimed span IDs are pruned idempotently. - receive exit with queued and in-flight emulator writes plus desktop and remote mappings; prove main delays `runtime.onPtyExit`, renderer exit, model disposal, and subscription cleanup until prior owners settle/transfer, and - prove deadline first cancels only the exited token, rejects late generation - callbacks, and reports incomplete restore-required exit; separately make the - cancellation proof fail and prove only then the provider closes; + prove deadline first acquires cancellation proof, transfers published + projections, commits proof, prepares/finalizes once, and rejects late + generation callbacks; close the generation while proof is pending and prove + no final exit publishes, then separately make proof fail while current and + prove only then the provider closes; - verify round-robin progress across 50 continuously active PTYs; - verify spans are freed only after all divergent cursors advance, cancel, or transfer; @@ -2371,7 +2611,8 @@ reported as headed paired-server, headless `orca serve`, remote-runtime lossless subscription, macOS node-pty, Windows ConPTY/named pipe, or WSL evidence. -Run paired-runtime coverage separately when Slice 5d maps remote consumers: +The Slice 5d mapping is implemented, but live paired-runtime coverage remains +a separate promotion requirement: | Topology | Required oracle | | ---------------------------------------------- | ------------------------------------------------------------------------------------------- | @@ -2423,16 +2664,18 @@ Implementation and rollout stages: reliability oracles. 2. Slices 5a-5c are implemented behind the startup gate and relay launch policy with unit/service contracts and prior direct Docker SSH evidence. -3. Slice 5d's remote source-range transaction is implemented at deterministic - main/runtime seams. Live headed paired, headless, WSL, Windows, - local-daemon, local-provider, and folder-workspace evidence remains future - work before those topologies are marked covered. -4. complete exact-head review, focused/full validation, and the direct Docker - rerun for the current activation/recovery/deploy diff; -5. run an internal same-build canary comparing gate-off and V1 sink, +3. Slices 5d-5e implement remote source-range transactions and the final + lifecycle reconciliation at deterministic main/runtime/provider/relay + seams. Live headed paired, headless, WSL, Windows, local-daemon, + local-provider, and folder-workspace evidence remains future work before + those topologies are marked covered. +4. Current-head deterministic reconciliation and focused integrated-fix checks + are complete. Full validation and a direct Docker rerun of the final + lifecycle diff remain required before promotion. +5. Run an internal same-build canary comparing gate-off and V1 sink, model-admission, latency, sealed-exit, recovery, and reconnect metrics while separately counting orphan prior-version daemons; -6. default on only after zero ordering/data-loss failures and stable memory +6. Default on only after zero ordering/data-loss failures and stable memory plateaus for one release cycle; remove the flag only after same-build legacy fallback and versioned-orphan cleanup stay healthy through the supported upgrade window.