mirror of
https://github.com/stablyai/orca.git
synced 2026-10-19 16:03:26 +00:00
fix(ssh): order exit cancellation cleanup
This commit is contained in:
@@ -3103,8 +3103,8 @@
|
||||
"https://github.com/stablyai/orca/pull/7002",
|
||||
"https://github.com/stablyai/orca/pull/7054"
|
||||
],
|
||||
"invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.",
|
||||
"oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, and ACK-gated in-flight bounds. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, exact early-ACK reservation through send settlement or same-token retry, recovery-completion settlement before live admission, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved gate-off primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.",
|
||||
"invariant": "High-volume terminal output must stay bounded across daemon socket writes, SSH relay writer/decoder/source retention/reconnect, main model and projection admission, runtime metadata, detectors, and tail checks, main-to-renderer in-flight bytes, renderer scheduler queues, and hidden-output restore without starving focused input. Negotiated SSH spans remain exact and contiguous through activation, exit, cancellation, recovery, desktop projection, and required remote replacement; exit-time cancellation transfers published projections before proof commit, and generation close fences late proof from final exit. Additional subscribers cannot stall the owner, while the required legacy primary retains backpressure.",
|
||||
"oracle": "The current executable slice injects main-process renderer backlog pressure, then asserts unchanged path-provenance history reuse for pathless output, ordinary terminal-wait detection without a joined-tail allocation and with full prompt-family continuity, per-PTY and total pending-output caps, preserved sequenced-tail metadata, active-pending protection ahead of background trimming, and ACK-gated in-flight bounds. The joined source-intake/multiplex oracle blocks renderer or headless serialization, admits one snapshot-covered span and one trailing span, requires replacement reservation only after the authoritative sequence is known, advances upstream ACK eligibility exactly to the covered boundary after SnapshotEnd, and delivers then ACKs the trailing span through the ordinary live path without cancellation. Source-range ledger and multiplex seams additionally require cumulative byte ACK 40 then 100 for one 100-byte frame to release exact byte credit without early source settlement, admit a contiguous higher-generation recovered token while the prior token remains unsettled, and reject stale stream or source generations. The exit-deadline oracle publishes a source-backed desktop projection, advances a fake clock to cancellation, and requires one projection transfer, proof commit, preparation, and final exit with no provider close or retained obligation; a second controlled proof remains pending across generation close and must never publish final exit. Direct-SSH contracts additionally prove activation-response settlement before the first source frame, exact early-ACK reservation through send settlement or same-token retry, recovery-completion settlement before live admission, invalid-checkpoint restore retention through response settlement followed by fresh-token retry and one live source frame, isolated saturated-subscriber eviction with healthy owner liveness, preserved gate-off primary backpressure, one-write ownership across write(false)/drain, control selection ahead of queued ordinary PTY input with FIFO lanes and bounded fairness, liveness rebasing, failed-exit retention through late ACK or exact recovery, response-settlement cancellation authority, token-scoped exit cancellation, exact checkpoint-to-recoveryEnd continuity, stale-owner one-shot retry, no physical PTY teardown on recovery failure, exact provider-generation closure and once-only cleanup when recovery cancellation publication or proof rejects, 16 MiB-plus-1 MiB decoder caps, charged per-PTY/session retention budgets, and eight-wide isolated reattach. The live Electron perf oracle adds hidden-output floods, renderer scheduler queue depth, dropped-output-zero normal scenarios, and active key latency budgets before promotion.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime-path-candidate-history.test.ts",
|
||||
@@ -3114,7 +3114,7 @@
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/relay/dispatcher.test.ts src/relay/pty-handler-source-publication.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-relay-session-recovery-races.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-source-range-ledger.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-channel-multiplexer-backpressure.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/agent-hooks/wsl-hook-relay-sentinel.test.ts --reporter=dot",
|
||||
"ORCA_E2E_SSH_DOCKER=1 ORCA_SSH_PTY_SOURCE_CREDIT_V1=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
|
||||
],
|
||||
@@ -3144,6 +3144,7 @@
|
||||
"src/relay/ssh-pty-source-credit-adapter.test.ts",
|
||||
"src/shared/pty-consumer-session.test.ts",
|
||||
"src/main/ipc/ssh-pty-output-intake.test.ts",
|
||||
"src/main/ipc/ssh-pty-output-exit-deadline.test.ts",
|
||||
"src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts",
|
||||
"src/main/runtime/rpc/terminal-source-range-ledger.test.ts",
|
||||
"src/main/runtime/rpc/terminal-multiplex.test.ts",
|
||||
@@ -3270,6 +3271,13 @@
|
||||
"desktop source identity and scanner facts commit atomically or roll back without credit"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/ipc/ssh-pty-output-exit-deadline.test.ts",
|
||||
"assertions": [
|
||||
"published projection transfer precedes cancellation-proof span reclamation with one preparation and final exit",
|
||||
"generation close fences a pending cancellation proof from final exit and retained obligations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts",
|
||||
"assertions": [
|
||||
@@ -3405,6 +3413,15 @@
|
||||
"durationSeconds": 7.17,
|
||||
"summary": "Three deterministic main/runtime files passed 78 tests, including joined renderer/headless snapshot admission, exact partial cumulative credit, and recovered-token continuity; no live paired server or Docker topology was exercised."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-28",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/shared/pty-consumer-session.test.ts src/main/ipc/ssh-pty-output-intake.test.ts src/main/ipc/ssh-pty-output-exit-deadline.test.ts src/main/ipc/ssh-pty-remote-source-range-consumers.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/ssh/ssh-multiplexer-transport-writer.test.ts src/main/ssh/ssh-channel-multiplexer.test.ts src/main/ssh/ssh-pty-source-credit-rollout.test.ts src/main/ssh/ssh-relay-deploy.test.ts src/relay/dispatcher-client-writer.test.ts --reporter=dot",
|
||||
"result": "passed",
|
||||
"durationSeconds": 7.05,
|
||||
"summary": "Ten deterministic source-intake, exit-deadline, remote-consumer, multiplex, writer, rollout, and deploy files passed 189 tests, including both exit-time ordering oracles."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-28",
|
||||
"runner": "local",
|
||||
@@ -3425,7 +3442,7 @@
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output is capped per PTY and in total, direct-SSH source/recovery memory and both frame decoders stay within exact caps, and 50-PTY reconnect isolates slow/failing siblings. The deterministic main-to-relay drain oracle failed with ordinary-2 and ordinary-3 ahead of a later control frame under the single FIFO, then passed with control-first lane selection and four-write ordinary fairness. Before the snapshot-admission fix, the focused consumer oracle failed because the trailing span entered transferring instead of remaining open; after retaining model-sequence ends and reserving at the serialized fence, the joined main/runtime oracle passed for renderer and headless sources with upstream boundaries 4 then 8. Before cumulative partial ACK and source-rotation support, the ledger rejected ACK 40 of 100 and the multiplex seam detached instead of publishing recovered token B; both exact oracles pass while stale stream and source generations remain rejected. Before restore-record retirement, invalid-checkpoint recovery followed by restore/retry returned restoreRequired a second time instead of opening a fresh activation; the same deterministic oracle passes after exact response-settlement cleanup. Other controls previously failed with serial reattach, UTF-8-only accounting, and roughly 32 MiB decoder retention. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion."
|
||||
"evidence": "Tests assert pathless runtime output reuses unchanged path-provenance history with zero old-candidate byte scans, ordinary saturated terminal tails retain no rebuilt wait text or repeated phrase scans while every blocked/ready prompt family remains live, main pending renderer output is capped per PTY and in total, direct-SSH source/recovery memory and both frame decoders stay within exact caps, and 50-PTY reconnect isolates slow/failing siblings. The deterministic main-to-relay drain oracle failed with ordinary-2 and ordinary-3 ahead of a later control frame under the single FIFO, then passed with control-first lane selection and four-write ordinary fairness. On exact pre-fix baseline db167ea3d with test-only oracles, cancellation proof reclaimed the published projection span before transfer and generation close allowed a late proof to publish final exit; both deterministic oracles pass after proof acquisition and commit are separated under the exact deadline barrier. Before the snapshot-admission fix, the focused consumer oracle failed because the trailing span entered transferring instead of remaining open; after retaining model-sequence ends and reserving at the serialized fence, the joined main/runtime oracle passed for renderer and headless sources with upstream boundaries 4 then 8. Before cumulative partial ACK and source-rotation support, the ledger rejected ACK 40 of 100 and the multiplex seam detached instead of publishing recovered token B; both exact oracles pass while stale stream and source generations remain rejected. Before restore-record retirement, invalid-checkpoint recovery followed by restore/retry returned restoreRequired a second time instead of opening a fresh activation; the same deterministic oracle passes after exact response-settlement cleanup. Other controls previously failed with serial reattach, UTF-8-only accounting, and roughly 32 MiB decoder retention. Needs daemon-stream contract coverage and broader hidden-output/input-latency perf artifacts before promotion."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RemoteTerminalSourceRangeStreamIdentity } from '../runtime/remote-terminal-source-range-consumer'
|
||||
import {
|
||||
SshPtyOutputIntake,
|
||||
type SshPtyOutputIntakeDependencies,
|
||||
type SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake'
|
||||
import type { SshPtySourceCancellationProof } from './ssh-pty-output-intake-contract'
|
||||
|
||||
type ExitDeadlineHarness = Readonly<{
|
||||
intake: SshPtyOutputIntake
|
||||
dependencies: SshPtyOutputIntakeDependencies
|
||||
cancelSourceDelivery: ReturnType<typeof vi.fn>
|
||||
exits: string[]
|
||||
}>
|
||||
|
||||
const stream: RemoteTerminalSourceRangeStreamIdentity = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
|
||||
function createHarness(
|
||||
cancelSourceDelivery: NonNullable<SshPtyOutputIntakeDependencies['cancelSourceDelivery']>
|
||||
): ExitDeadlineHarness {
|
||||
const exits: string[] = []
|
||||
const cancellation = vi.fn(cancelSourceDelivery)
|
||||
const dependencies: SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: () => 0,
|
||||
acceptModel: (event) => ({ sequence: event.rawLength, completion: Promise.resolve() }),
|
||||
project: vi.fn(),
|
||||
prepareExit: vi.fn(),
|
||||
finalizeExit: () => exits.push('exit'),
|
||||
pauseProvider: vi.fn(() => true),
|
||||
resumeProvider: vi.fn(),
|
||||
closeProvider: vi.fn(),
|
||||
cancelSourceDelivery: cancellation
|
||||
}
|
||||
return {
|
||||
intake: new SshPtyOutputIntake(dependencies, {
|
||||
exitBarrierMs: 10,
|
||||
exitCancellationProofMs: 100
|
||||
}),
|
||||
dependencies,
|
||||
cancelSourceDelivery: cancellation,
|
||||
exits
|
||||
}
|
||||
}
|
||||
|
||||
async function publishSource(harness: ExitDeadlineHarness): Promise<SshPtyOutputReceipt> {
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
expect(remote.attach(stream)).toBe(true)
|
||||
const receipt = await harness.intake.acceptData({
|
||||
id: 'pty-1',
|
||||
data: 'aaaa',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
harness.intake.publishProjectionPrefix([receipt.projection.identity.projectionSemanticsId], 4, 4)
|
||||
return receipt
|
||||
}
|
||||
|
||||
function acceptExit(harness: ExitDeadlineHarness) {
|
||||
return harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
}
|
||||
|
||||
describe('SshPtyOutputExitDeadline', () => {
|
||||
it('transfers published projections before cancellation proof reclaims their spans', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const harness = createHarness(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
|
||||
const receipt = await publishSource(harness)
|
||||
const exitResult = acceptExit(harness).then(
|
||||
() => ({ ok: true as const }),
|
||||
(error: Error) => ({ ok: false as const, error })
|
||||
)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
|
||||
expect(await exitResult).toEqual({ ok: true })
|
||||
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.dependencies.prepareExit).toHaveBeenCalledOnce()
|
||||
expect(harness.exits).toEqual(['exit'])
|
||||
expect(harness.intake.getDebugSnapshot()).toMatchObject({
|
||||
projection: { records: 0 },
|
||||
source: { openedTokens: 0, ptyIdentities: 0 },
|
||||
exitBarriers: 0
|
||||
})
|
||||
expect(() =>
|
||||
harness.intake
|
||||
.getRemoteSourceRangeConsumerHooks()
|
||||
.settle(stream, [receipt.projection.desktopSpan!])
|
||||
).not.toThrow()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('generation close fences a pending cancellation proof from final exit', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let resolveCancellation!: (proof: SshPtySourceCancellationProof) => void
|
||||
const cancellation = new Promise<SshPtySourceCancellationProof>((resolve) => {
|
||||
resolveCancellation = resolve
|
||||
})
|
||||
const harness = createHarness(() => cancellation)
|
||||
await publishSource(harness)
|
||||
const exitResult = acceptExit(harness).then(
|
||||
() => ({ ok: true as const }),
|
||||
(error: Error) => ({ ok: false as const, error })
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
const closed = await exitResult
|
||||
expect(closed.ok).toBe(false)
|
||||
expect(closed.ok ? '' : closed.error.message).toContain('provider-closed')
|
||||
resolveCancellation({ sentEndSu: 4, creditedEndSu: 0 })
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(harness.exits).toEqual([])
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getDebugSnapshot()).toMatchObject({
|
||||
projection: { records: 0 },
|
||||
source: { openedTokens: 0, ptyIdentities: 0 },
|
||||
exitBarriers: 0
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -5,7 +5,10 @@ import type {
|
||||
SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation'
|
||||
import type { SshPtyOutputSourceObligations } from './ssh-pty-output-source-obligations'
|
||||
import type {
|
||||
SshPtyOutputSourceObligations,
|
||||
SshPtySourceCancellationProofCommit
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
|
||||
type SshPtyOutputExitDeadlineDependencies = Readonly<{
|
||||
admission: SshPtyModelAdmission
|
||||
@@ -18,6 +21,8 @@ type SshPtyOutputExitDeadlineDependencies = Readonly<{
|
||||
|
||||
export class SshPtyOutputExitDeadline {
|
||||
private readonly barriersByGeneration = new Map<number, Set<SshPtyExitBarrier>>()
|
||||
private readonly cancellationOwnedExits = new Set<string>()
|
||||
private readonly preparedExits = new Set<string>()
|
||||
private readonly barrierMs: number
|
||||
private readonly cancellationProofMs: number
|
||||
|
||||
@@ -46,7 +51,8 @@ export class SshPtyOutputExitDeadline {
|
||||
const barrier: SshPtyExitBarrier = {
|
||||
timer: setTimeout(() => {
|
||||
timeoutStarted = true
|
||||
void this.cancelTimedOutExit(event).then(
|
||||
this.cancellationOwnedExits.add(this.exitKey(event))
|
||||
void this.cancelTimedOutExit(event, barrier).then(
|
||||
() => settle({ ok: true }),
|
||||
(error) => {
|
||||
this.dependencies.intake.closeProvider?.(
|
||||
@@ -86,14 +92,39 @@ export class SshPtyOutputExitDeadline {
|
||||
|
||||
closeGeneration(providerGeneration: number, error: Error): void {
|
||||
const barriers = this.barriersByGeneration.get(providerGeneration)
|
||||
if (!barriers) {
|
||||
if (barriers) {
|
||||
for (const barrier of barriers) {
|
||||
clearTimeout(barrier.timer)
|
||||
barrier.reject(error)
|
||||
}
|
||||
this.barriersByGeneration.delete(providerGeneration)
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.cancellationOwnedExits) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.cancellationOwnedExits.delete(key)
|
||||
}
|
||||
}
|
||||
for (const key of this.preparedExits) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.preparedExits.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validateNormalExit(event: SshPtyOutputExitEvent): void {
|
||||
if (this.cancellationOwnedExits.has(this.exitKey(event))) {
|
||||
throw outputIntakeError('ssh_exit_delivery_canceled')
|
||||
}
|
||||
}
|
||||
|
||||
prepareExitOnce(event: SshPtyOutputExitEvent): void {
|
||||
const key = this.exitKey(event)
|
||||
if (this.preparedExits.has(key)) {
|
||||
return
|
||||
}
|
||||
for (const barrier of barriers) {
|
||||
clearTimeout(barrier.timer)
|
||||
barrier.reject(error)
|
||||
}
|
||||
this.barriersByGeneration.delete(providerGeneration)
|
||||
this.preparedExits.add(key)
|
||||
this.dependencies.intake.prepareExit(event)
|
||||
}
|
||||
|
||||
get activeBarriers(): number {
|
||||
@@ -103,7 +134,10 @@ export class SshPtyOutputExitDeadline {
|
||||
)
|
||||
}
|
||||
|
||||
private async cancelTimedOutExit(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
private async cancelTimedOutExit(
|
||||
event: SshPtyOutputExitEvent,
|
||||
barrier: SshPtyExitBarrier
|
||||
): Promise<void> {
|
||||
const cancel = this.dependencies.intake.cancelSourceDelivery
|
||||
if (!cancel) {
|
||||
throw outputIntakeError('ssh_source_cancellation_publisher_unavailable')
|
||||
@@ -113,16 +147,35 @@ export class SshPtyOutputExitDeadline {
|
||||
'ssh_exit_delivery_canceled'
|
||||
)
|
||||
this.dependencies.sourceObligations.sealPty(event)
|
||||
const cancellation = this.dependencies.sourceObligations.cancelPty(event, (request) =>
|
||||
cancel(event.providerGeneration, request)
|
||||
const cancellation = this.dependencies.sourceObligations.requestPtyCancellationProof(
|
||||
event,
|
||||
(request) => cancel(event.providerGeneration, request)
|
||||
)
|
||||
const canceled = await this.withCancellationProofDeadline(cancellation)
|
||||
if (!canceled) {
|
||||
let commit: SshPtySourceCancellationProofCommit | null
|
||||
try {
|
||||
commit = await this.withCancellationProofDeadline(cancellation)
|
||||
} catch (error) {
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
if (!commit) {
|
||||
throw outputIntakeError('ssh_source_cancellation_identity_unavailable')
|
||||
}
|
||||
this.dependencies.projections.transferPty(event.id, 'ssh-exit-delivery-canceled')
|
||||
this.dependencies.intake.prepareExit(event)
|
||||
this.dependencies.sourceObligations.commitPtyCancellationProof(commit)
|
||||
this.prepareExitOnce(event)
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
this.dependencies.intake.finalizeExit(event)
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
this.dependencies.projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
@@ -131,6 +184,14 @@ export class SshPtyOutputExitDeadline {
|
||||
)
|
||||
}
|
||||
|
||||
private isActive(providerGeneration: number, barrier: SshPtyExitBarrier): boolean {
|
||||
return this.barriersByGeneration.get(providerGeneration)?.has(barrier) ?? false
|
||||
}
|
||||
|
||||
private exitKey(event: SshPtyOutputExitEvent): string {
|
||||
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
|
||||
}
|
||||
|
||||
private withCancellationProofDeadline<T>(promise: Promise<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
|
||||
@@ -11,6 +11,7 @@ export async function settleSshPtyOutputExit(args: {
|
||||
projections: SshPtyLegacyProjectionLedger
|
||||
dependencies: SshPtyOutputIntakeDependencies
|
||||
validateGeneration: () => void
|
||||
prepareExit?: () => void
|
||||
afterAdmissionIdle?: () => void
|
||||
waitForSourceTerminal?: () => Promise<void>
|
||||
beforeFinalize?: () => void
|
||||
@@ -21,6 +22,7 @@ export async function settleSshPtyOutputExit(args: {
|
||||
projections,
|
||||
dependencies,
|
||||
validateGeneration,
|
||||
prepareExit,
|
||||
afterAdmissionIdle,
|
||||
waitForSourceTerminal,
|
||||
beforeFinalize
|
||||
@@ -32,7 +34,11 @@ export async function settleSshPtyOutputExit(args: {
|
||||
validateGeneration()
|
||||
afterAdmissionIdle?.()
|
||||
try {
|
||||
dependencies.prepareExit(event)
|
||||
if (prepareExit) {
|
||||
prepareExit()
|
||||
} else {
|
||||
dependencies.prepareExit(event)
|
||||
}
|
||||
} catch (error) {
|
||||
projections.closePty(
|
||||
event.id,
|
||||
|
||||
@@ -171,7 +171,11 @@ export class SshPtyOutputIntake {
|
||||
admission: this.admission,
|
||||
projections: this.projections,
|
||||
dependencies: this.dependencies,
|
||||
validateGeneration: () => this.generationGuard.validate(event),
|
||||
validateGeneration: () => {
|
||||
this.generationGuard.validate(event)
|
||||
this.exitDeadline.validateNormalExit(event)
|
||||
},
|
||||
prepareExit: () => this.exitDeadline.prepareExitOnce(event),
|
||||
afterAdmissionIdle: () => this.sourceObligations.sealPty(event),
|
||||
waitForSourceTerminal: () => this.sourceObligations.whenPtyTerminal(event),
|
||||
beforeFinalize: () => this.sourceObligations.markExitPublished(event)
|
||||
|
||||
@@ -22,6 +22,11 @@ export type SshPtyOutputSourceReservation = Readonly<{
|
||||
span: PtySourceSpan
|
||||
}>
|
||||
|
||||
export type SshPtySourceCancellationProofCommit = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
proof: SshPtySourceCancellationProof
|
||||
}>
|
||||
|
||||
export type SshPtyAcceptedSourceCheckpoint = Readonly<{
|
||||
id: string
|
||||
providerGeneration: number
|
||||
@@ -145,18 +150,21 @@ export class SshPtyOutputSourceObligations {
|
||||
return identity ? this.coordinator.whenTerminal(identity) : Promise.resolve()
|
||||
}
|
||||
|
||||
async cancelPty(
|
||||
async requestPtyCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
cancel: (request: SshPtySourceCancellationRequest) => Promise<SshPtySourceCancellationProof>
|
||||
): Promise<boolean> {
|
||||
): Promise<SshPtySourceCancellationProofCommit | null> {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (!identity) {
|
||||
return false
|
||||
return null
|
||||
}
|
||||
const request = this.coordinator.beginExitTimeout(identity)
|
||||
const proof = await cancel(request)
|
||||
this.coordinator.applyCancellationProof(identity, proof)
|
||||
return true
|
||||
return Object.freeze({ identity, proof })
|
||||
}
|
||||
|
||||
commitPtyCancellationProof(commit: SshPtySourceCancellationProofCommit): void {
|
||||
this.coordinator.applyCancellationProof(commit.identity, commit.proof)
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
|
||||
Reference in New Issue
Block a user