From 7327640d3a2b1f0d6aa11a3e8c0fc2dea7f63c8d Mon Sep 17 00:00:00 2001 From: OrcaWin <293788423+OrcaWin@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:24:47 -0700 Subject: [PATCH] fix(ssh): contain migration failure per PTY --- config/reliability-gates.jsonc | 10 +- .../2026-07-27-ssh-relay-pty-backpressure.md | 36 +++++-- .../ipc/ssh-pty-model-admission-migration.ts | 35 ++++++ src/main/ipc/ssh-pty-model-admission.test.ts | 26 +++++ src/main/ipc/ssh-pty-model-admission.ts | 40 ++++--- .../ssh-pty-output-model-migration.test.ts | 101 ++++++++++++++++++ 6 files changed, 214 insertions(+), 34 deletions(-) diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 7402cc7e1ef..e1558bf3f15 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -3089,8 +3089,8 @@ "https://github.com/stablyai/orca/pull/7002", "https://github.com/stablyai/orca/pull/7054" ], - "invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; provisional activation data cannot project, reconnect may release it only into the exact attempt's private recovery quarantine after contract validation, and exit seals that quarantine against later same-token frames until ordered admission or exact cancellation proof. Reconnect checkpoint capture freezes exact provider-generation/PTy admission, cancels queued old work, and waits for the running raw completion before exposing its accepted checkpoint; timeout or failure detaches only that PTy's old model, makes the checkpoint unavailable, and releases the bounded fence, while overlapping reconnects preserve any earlier outstanding fence. Pending renderer projection IDs are capped and compacted across split remainders, exit preparation owns its renderer fence through finalization, exit-time cancellation transfers published projections before proof commit, generation close fences late proof while draining exact projection waiters, closed-generation identity compacts without weakening stale rejection, canceled source-delivery retirement retains at most one ordered token per PTY, and pause/resume targets only the exact provider generation. Additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.", - "oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, a 1,024-ID projection cap with split-remainder compaction, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, exact provider-generation pause ownership, renderer exit/data ordering, and ACK-gated in-flight bounds. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations without releasing in-flight byte credit. Replacement reservation rolls back every earlier span on a later failure, commit requires the exact frozen transfer states, and an authoritatively reclaimed covered span rejects commit without local trim authority while rollback removes the reservation idempotently. The exit-deadline oracle publishes a source-backed desktop projection, advances a fake clock to cancellation, and requires one projection transfer, proof commit, preparation, and final exit with no provider close, retained obligation, duplicate finalization, or process-lifetime cancellation tombstone; a second controlled proof remains pending across generation close and must never publish final exit, while both paths release the renderer preparation lease exactly once. The model-migration oracle accepts span A, blocks span B's raw emulator completion, begins disconnect migration, and proves no checkpoint or reconnect attach can remain at A; releasing B advances the exact checkpoint to B and projects B once. Its fake-clock arm reaches the 10-second deadline, requires checkpoint-unavailable, one exact model reset, zero retained admission charge and timers, and no effect from late raw settlement. The intake additionally closes 2,048 sequential provider generations into one exact range, rejects stale events across that range, and preserves an out-of-order live gap until its own close. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, claim-gated projection under same-decoder-turn response/data delivery, exact early-ACK reservation through send settlement or same-token retry, recovery completion visible before lease-held frames transfers those frames only into the private quarantine and admits no output until exact checkpoint-to-recoveryEnd coverage, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved gate-off primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.", + "invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; provisional activation data cannot project, reconnect may release it only into the exact attempt's private recovery quarantine after contract validation, and exit seals that quarantine against later same-token frames until ordered admission or exact cancellation proof. Reconnect checkpoint capture freezes exact provider-generation/PTy admission, cancels queued old work, and waits for the running raw completion before exposing its accepted checkpoint; timeout or failure detaches only that PTy's old model, makes the checkpoint unavailable, and releases the bounded fence without closing the shared provider, while the same failure outside an active exact migration remains generation-fatal and overlapping reconnects preserve any earlier outstanding fence. Pending renderer projection IDs are capped and compacted across split remainders, exit preparation owns its renderer fence through finalization, exit-time cancellation transfers published projections before proof commit, generation close fences late proof while draining exact projection waiters, closed-generation identity compacts without weakening stale rejection, canceled source-delivery retirement retains at most one ordered token per PTY, and pause/resume targets only the exact provider generation. Additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.", + "oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, a 1,024-ID projection cap with split-remainder compaction, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, exact provider-generation pause ownership, renderer exit/data ordering, and ACK-gated in-flight bounds. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations without releasing in-flight byte credit. Replacement reservation rolls back every earlier span on a later failure, commit requires the exact frozen transfer states, and an authoritatively reclaimed covered span rejects commit without local trim authority while rollback removes the reservation idempotently. The exit-deadline oracle publishes a source-backed desktop projection, advances a fake clock to cancellation, and requires one projection transfer, proof commit, preparation, and final exit with no provider close, retained obligation, duplicate finalization, or process-lifetime cancellation tombstone; a second controlled proof remains pending across generation close and must never publish final exit, while both paths release the renderer preparation lease exactly once. The model-migration oracle accepts span A, blocks span B's raw emulator completion, begins disconnect migration, and proves no checkpoint or reconnect attach can remain at A; releasing B advances the exact checkpoint to B and projects B once. Its fake-clock arm reaches the 10-second deadline, requires checkpoint-unavailable, one exact model reset, zero retained admission charge and timers, and no effect from late raw settlement. Its two-PTY failure arm rejects one running raw callback under an active exact migration and requires one checkpoint-unavailable result, one model reset, zero provider closes, retained sibling checkpoint and transport liveness, zero charges and timers, and an unaffected other generation; a non-migrating sibling oracle still requires generation close. The intake additionally closes 2,048 sequential provider generations into one exact range, rejects stale events across that range, and preserves an out-of-order live gap until its own close. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, claim-gated projection under same-decoder-turn response/data delivery, exact early-ACK reservation through send settlement or same-token retry, recovery completion visible before lease-held frames transfers those frames only into the private quarantine and admits no output until exact checkpoint-to-recoveryEnd coverage, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved gate-off primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-pending-projection-admissions.test.ts src/main/ipc/ssh-pty-legacy-projection.test.ts src/main/ipc/ssh-pty-model-admission.test.ts --reporter=dot", @@ -3350,7 +3350,8 @@ "file": "src/main/ipc/ssh-pty-model-admission.test.ts", "assertions": [ "disposal resumes every paused provider generation exactly once, including pause-only rejected entries", - "migration cancels queued old-generation work while retaining the one running raw completion" + "migration cancels queued old-generation work while retaining the one running raw completion", + "a callback failure outside migration closes its generation and rejects a sibling PTY admission" ] }, { @@ -3367,7 +3368,8 @@ "file": "src/main/ipc/ssh-pty-output-model-migration.test.ts", "assertions": [ "migration waits for blocked span B before exporting its checkpoint and projects B exactly once", - "migration timeout returns checkpoint-unavailable, resets one exact PTY model, and releases admission charge and timers before late raw settlement" + "migration timeout returns checkpoint-unavailable, resets one exact PTY model, and releases admission charge and timers before late raw settlement", + "migration-owned callback failure resets only the failed PTY while late settlement cannot advance it and sibling checkpoint, provider transport, and another generation remain live" ] }, { diff --git a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md index 053b2582f70..8cfca550bf8 100644 --- a/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md +++ b/docs/plans/2026-07-27-ssh-relay-pty-backpressure.md @@ -824,12 +824,15 @@ an empty newer generation cannot expose the earlier checkpoint. Before the deadline, every old callback remains owned by its admission and generation until raw completion settles. Stale-owner retry invalidates the checkpoint value but retains the same fence, then requests restore after it settles. If -the 10-second migration deadline expires or -the callback fails, main cancels only that admission, detaches that PTY's old headless model, -prefers provider/renderer snapshot restore, and requests -`checkpointUnavailable`; recovery reports `restoreRequired` instead of -replaying a guessed gap. Late raw completion stays on the detached model and -cannot advance the canceled source checkpoint. +the 10-second migration deadline expires or the callback fails, admission +checks the exact active migration before its generation-fatal default. Main +cancels only that admission, detaches that PTY's old headless model, prefers +provider/renderer snapshot restore, and requests `checkpointUnavailable`; +recovery reports `restoreRequired` instead of replaying a guessed gap. The +shared mux, sibling PTYs, filesystem, and Git traffic remain live. Outside an +active exact-PTY migration, the same callback failure still closes its provider +generation. Late raw completion stays on the detached model and cannot advance +the canceled source checkpoint. Attach sends `sourceRecovery: { status: 'checkpoint', clientGeneration, ownerGeneration, ptyIncarnation, deliveryToken, acceptedSourceEndSu }`. A @@ -1413,9 +1416,11 @@ queued-but-unstarted entries and reject their receipts exactly once. An in-flight emulator callback owns its entry until completion or failure and must pass its captured token/generation check before committing. Reconnect and supersession use the migration fence above rather than letting that callback -race a replacement checkpoint. The old Promise chain may remain as the -per-PTY execution primitive only after each link is charged by this scheduler; -it is no longer an unbounded owner. +race a replacement checkpoint. Failure dispatch consults the exact +`(providerGeneration, ptyId)` migration owner before invoking generation close; +this does not suppress the ordinary non-migrating failure policy. The old +Promise chain may remain as the per-PTY execution primitive only after each +link is charged by this scheduler; it is no longer an unbounded owner. ### Desktop, mobile, web, and agent policy @@ -2037,6 +2042,9 @@ transition: - close each per-generation model-migration gate after its settlement, remove its per-PTY fence and timer exactly once, and retain any earlier outstanding per-PTY fence across an overlapping reconnect; +- on migration-owned callback failure, release only that PTY's admission + charge, timer, and tracked completion before reset/restore; do not close the + shared provider or sibling admissions; - replace the latest canceled-token record per PTY on ordered cancellation and clear it on exact next activation, PTY exit, or provider teardown; - release shared spans no longer referenced; @@ -2366,6 +2374,9 @@ boundaries even though they ship in one PR. `SshRelaySession` awaits the returned per-PTY Promise before constructing `sourceRecovery`, and `OrcaRuntimeService` detaches only the failed PTY's old headless model before snapshot-backed restore. +- `SshPtyModelAdmission` retains generation-fatal callback handling except when + the exact key is actively migrating; `SshPtyOutputIntake` uses the same key + to keep provider close from escaping that migration owner. - Reconnect V1 remains under the startup gate; its late-ACK, timeout, supersession, and generation-close deterministic oracles pass. @@ -2528,6 +2539,13 @@ lease-held body frames, and fail-closed exact-generation cleanup. A separate PTY. Transferred recovery data remains private until body/fence validation, and a proved token-local failure causes no physical PTY shutdown, ownership delete, or lease expiry. +The model-migration failure oracle uses two PTYs in one provider generation: +one raw callback rejects under an active fence, returns +`checkpointUnavailable`, resets one model, and releases its charge and timer; +the sibling checkpoint, provider mux, filesystem/Git transport, and another +provider generation remain live. A paired admission oracle proves the same +failure without an active migration still closes the generation and rejects +its sibling receipt. The remote-consumer seam covers partial cumulative ACK, safe higher-generation token rotation, and token cancellation followed by late detach/replacement commit, proving reclaimed span IDs reject commit, preserve rollback ownership, diff --git a/src/main/ipc/ssh-pty-model-admission-migration.ts b/src/main/ipc/ssh-pty-model-admission-migration.ts index 917deeb5f97..fc860f4716c 100644 --- a/src/main/ipc/ssh-pty-model-admission-migration.ts +++ b/src/main/ipc/ssh-pty-model-admission-migration.ts @@ -42,6 +42,41 @@ export function closeSshPtyModelAdmissionMigrations( } } +export function settleSshPtyModelAdmissionFailure(args: { + id: string + usage: PtyUsage + entry: AdmissionEntry + error: Error + migratingPtys: ReadonlySet + closingGenerations: Set + release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void + closeGeneration: (providerGeneration: number) => void + cleanup: (id: string, usage: PtyUsage) => void +}): void { + if (args.entry.state !== 'running' || args.usage.running !== args.entry) { + return + } + const migrationOwnsFailure = args.migratingPtys.has(args.id) + if (!migrationOwnsFailure) { + args.closingGenerations.add(args.entry.key.providerGeneration) + } + args.usage.running = null + args.entry.state = 'settled' + args.release(args.entry.key, args.entry.charge) + args.entry.reject(migrationOwnsFailure ? migrationCompletionError(args.error) : args.error) + if (!migrationOwnsFailure) { + args.closeGeneration(args.entry.key.providerGeneration) + } + args.cleanup(args.id, args.usage) +} + +function migrationCompletionError(cause: Error): Error { + return Object.assign(new Error(cause.message), { + code: 'ssh_model_migration_completion_failed', + cause + }) +} + function cancelQueuedEntry( entry: AdmissionEntry, error: Error, diff --git a/src/main/ipc/ssh-pty-model-admission.test.ts b/src/main/ipc/ssh-pty-model-admission.test.ts index eed738f600c..bda8512601d 100644 --- a/src/main/ipc/ssh-pty-model-admission.test.ts +++ b/src/main/ipc/ssh-pty-model-admission.test.ts @@ -65,6 +65,32 @@ describe('SshPtyModelAdmission', () => { expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) }) + it('keeps non-migrating callback failure generation-fatal across sibling PTYs', async () => { + const admission = new SshPtyModelAdmission() + const failedCompletion = deferred() + const siblingCompletion = deferred() + const failed = accept(admission, failedCompletion.promise) + const sibling = admission.accept({ ptyId: 'pty-2', providerGeneration: 7 }, 'data', 4, () => ({ + sequence: 4, + completion: siblingCompletion.promise + })) + + failedCompletion.reject(new Error('emulator failed')) + + await expect(failed).rejects.toThrow('emulator failed') + await expect(sibling).rejects.toThrow('ssh_model_admission_completion_failed') + await expect( + admission.accept({ ptyId: 'pty-3', providerGeneration: 7 }, 'data', 4, () => ({ + sequence: 4, + completion: Promise.resolve() + })) + ).rejects.toThrow('ssh_model_admission_generation_closed') + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + siblingCompletion.resolve() + await Promise.resolve() + expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 }) + }) + it('resumes every paused provider generation exactly once on disposal', async () => { const resumeProvider = vi.fn() const admission = new SshPtyModelAdmission({ diff --git a/src/main/ipc/ssh-pty-model-admission.ts b/src/main/ipc/ssh-pty-model-admission.ts index 1034a8f9685..69a9036e2bd 100644 --- a/src/main/ipc/ssh-pty-model-admission.ts +++ b/src/main/ipc/ssh-pty-model-admission.ts @@ -15,18 +15,11 @@ import { type PtyUsage } from './ssh-pty-model-admission-entry' import { resolveSshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits' -import { - beginSshPtyModelAdmissionMigration, - closeSshPtyModelAdmissionMigrations -} from './ssh-pty-model-admission-migration' +import * as modelAdmissionMigration from './ssh-pty-model-admission-migration' import { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure' import { sshPtyModelAdmissionSnapshot } from './ssh-pty-model-admission-snapshot' -export type { - SshPtyModelAdmissionKey, - SshPtyModelAdmissionOptions, - SshPtyModelAdmissionReceipt -} from './ssh-pty-model-admission-contract' +export type * from './ssh-pty-model-admission-contract' export class SshPtyModelAdmission { private readonly limits: ReturnType @@ -99,7 +92,10 @@ export class SshPtyModelAdmission { release: (key, charge) => this.release(key, charge) }) ) - closeSshPtyModelAdmissionMigrations(this.migratingPtys, providerGeneration) + modelAdmissionMigration.closeSshPtyModelAdmissionMigrations( + this.migratingPtys, + providerGeneration + ) const generationPrefix = `${providerGeneration}\0` for (const id of this.idleWaiters.keys()) { if (id.startsWith(generationPrefix)) { @@ -109,7 +105,7 @@ export class SshPtyModelAdmission { } beginMigration(key: SshPtyModelAdmissionKey): void { - beginSshPtyModelAdmissionMigration({ + modelAdmissionMigration.beginSshPtyModelAdmissionMigration({ key, migratingPtys: this.migratingPtys, pressure: this.pressure, @@ -261,16 +257,18 @@ export class SshPtyModelAdmission { } private failEntry(id: string, usage: PtyUsage, entry: AdmissionEntry, error: Error): void { - if (entry.state !== 'running' || usage.running !== entry) { - return - } - this.closingGenerations.add(entry.key.providerGeneration) - usage.running = null - entry.state = 'settled' - this.release(entry.key, entry.charge) - entry.reject(error) - this.closeGeneration(entry.key.providerGeneration, 'ssh_model_admission_completion_failed') - this.cleanupUsage(id, usage) + modelAdmissionMigration.settleSshPtyModelAdmissionFailure({ + id, + usage, + entry, + error, + migratingPtys: this.migratingPtys, + closingGenerations: this.closingGenerations, + release: (key, charge) => this.release(key, charge), + closeGeneration: (providerGeneration) => + this.closeGeneration(providerGeneration, 'ssh_model_admission_completion_failed'), + cleanup: (entryId, entryUsage) => this.cleanupUsage(entryId, entryUsage) + }) } private finishEntry(id: string, usage: PtyUsage, entry: AdmissionEntry): boolean { diff --git a/src/main/ipc/ssh-pty-output-model-migration.test.ts b/src/main/ipc/ssh-pty-output-model-migration.test.ts index b4099754e53..e8007a0c274 100644 --- a/src/main/ipc/ssh-pty-output-model-migration.test.ts +++ b/src/main/ipc/ssh-pty-output-model-migration.test.ts @@ -93,4 +93,105 @@ describe('SshPtyOutputModelMigration', () => { vi.useRealTimers() } }) + + it('contains a running callback failure to its migrating PTY', async () => { + vi.useFakeTimers() + try { + const resetModelForMigration = vi.fn() + const harness = createHarness({ resetModelForMigration }) + const sibling = harness.intake.acceptData( + event({ + id: 'pty-sibling', + ptyIncarnation: 'incarnation-sibling', + source: { + spanId: 'span-sibling', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-sibling', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + harness.completions[0]!.resolve() + await sibling + const failed = harness.intake.acceptData( + event({ + id: 'pty-failed', + ptyIncarnation: 'incarnation-failed', + source: { + spanId: 'span-failed', + clientGeneration: 3, + ownerGeneration: 4, + deliveryToken: 'delivery-failed', + sourceStartSu: 0, + sourceEndSu: 4 + } + }) + ) + const migration = harness.intake.beginGenerationMigration(1) + + harness.completions[1]!.reject(new Error('emulator failed')) + + await expect(failed).rejects.toMatchObject({ + message: 'emulator failed', + code: 'ssh_model_migration_completion_failed' + }) + await expect(migration.byPty.get('pty-failed')).resolves.toEqual({ + status: 'checkpoint-unavailable', + reason: 'completion-failed' + }) + await expect(migration.byPty.get('pty-sibling')).resolves.toMatchObject({ + status: 'settled', + checkpoint: { id: 'pty-sibling', acceptedSourceEndSu: 4 } + }) + await migration.completion + expect(resetModelForMigration).toHaveBeenCalledTimes(1) + expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-failed') + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: 'pty-failed', acceptedSourceEndSu: 0 }), + expect.objectContaining({ id: 'pty-sibling', acceptedSourceEndSu: 4 }) + ]) + ) + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 0, + bytes: 0, + pressureFrames: 0, + migratingPtys: 2 + }) + expect(vi.getTimerCount()).toBe(0) + + harness.completions[1]!.resolve() + await Promise.resolve() + expect(resetModelForMigration).toHaveBeenCalledTimes(1) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + expect( + harness.intake + .getAcceptedSourceCheckpoints(1) + .find((checkpoint) => checkpoint.id === 'pty-failed')?.acceptedSourceEndSu + ).toBe(0) + + const unrelated = harness.intake.acceptData( + event({ + id: 'pty-unrelated', + providerGeneration: 2, + ptyIncarnation: 'incarnation-unrelated' + }) + ) + harness.completions[2]!.resolve() + await expect(unrelated).resolves.toMatchObject({ providerGeneration: 2 }) + expect(harness.dependencies.closeProvider).not.toHaveBeenCalled() + harness.intake.closeGeneration(1, 'connection_lost') + harness.intake.closeGeneration(2, 'connection_lost') + expect(harness.intake.getDebugSnapshot().model).toMatchObject({ + sourceUnits: 0, + bytes: 0, + migratingPtys: 0 + }) + } finally { + vi.useRealTimers() + } + }) })