feat(ssh): bound relay PTY output end to end (#11005)

* docs: design SSH relay PTY backpressure

* fix(ssh): bound relay frame decoding

* fix(relay): bound PTY output publication

* fix(ssh): bound PTY model admission

* fix(ssh): settle closed model admissions

* feat(ssh): negotiate bounded PTY consumer sessions

* fix(ssh): fence exit on renderer settlement

* feat(ssh): track PTY source credit end to end

* fix(ssh): recover bounded PTY output across reconnect

* feat(ssh): complete relay PTY output backpressure

* fix(ssh): close final PTY source credit races

* docs(ssh): record final backpressure validation

* feat(ssh): complete relay PTY source-credit lifecycle

* test(ssh): complete provider notification fixture

* fix(ssh): preserve terminal source credit across rotation

* fix(ssh): fail closed on recovery cancellation

* fix(ssh): prioritize mux control writes after drain

* fix(ssh): retire canceled relay restore deliveries

* fix(ssh): order exit cancellation cleanup

* fix(ssh): gate provisional source activation

* test(ssh): register mux drain-priority coverage

* fix(ssh): type stale owner recovery mismatches

* fix(ssh): close projection replacement races

* fix(relay): contain streaming edge failures

* fix(ssh): secure relay endpoint credentials

* docs(ssh): reconcile final backpressure lifecycle

* fix(ssh): bound main IPC output lifecycle

* fix(ssh): close recovery ownership gaps

* docs(ssh): record exact artifact validation

* fix(ssh): reject reclaimed snapshot replacements

* fix(ssh): fence model admission across reconnect

* fix(ssh): contain migration failure per PTY

* docs(ssh): record final exact-head validation

* test(ssh): align deploy fixtures with credential publication

* feat(ssh): add per-target bounded output setting

* fix(ssh): close source recovery review gaps

* fix(ssh): latch source credit environment override

* feat(ssh): make PTY source credit the default

* docs(ssh): record always-on relay validation

* docs(ssh): bind validation to current main

* test(ssh): grant source credit in IPC fixture

* test(ssh): grant source credit in fake relay

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
Jinjing
2026-07-29 17:03:15 -07:00
committed by GitHub
co-authored by OrcaWin
parent c676b6aa3b
commit 5f7807497e
189 changed files with 34164 additions and 2410 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -16,14 +16,20 @@ type FakeChild = ChildProcessWithoutNullStreams & { kill: ReturnType<typeof vi.f
function fakeChild(): FakeChild {
const child = new EventEmitter() as EventEmitter & {
stdout: EventEmitter
stdout: EventEmitter & {
pause: ReturnType<typeof vi.fn>
resume: ReturnType<typeof vi.fn>
}
stderr: EventEmitter
stdin: { write: ReturnType<typeof vi.fn> }
stdin: EventEmitter & { write: ReturnType<typeof vi.fn> }
kill: ReturnType<typeof vi.fn>
}
child.stdout = new EventEmitter()
child.stdout = Object.assign(new EventEmitter(), {
pause: vi.fn(),
resume: vi.fn()
})
child.stderr = new EventEmitter()
child.stdin = { write: vi.fn(() => true) }
child.stdin = Object.assign(new EventEmitter(), { write: vi.fn(() => true) })
child.kill = vi.fn()
return child as unknown as FakeChild
}
@@ -62,6 +68,10 @@ describe('waitForWslRelaySentinel', () => {
const transport = await promise
expect(typeof transport.write).toBe('function')
expect(typeof transport.onData).toBe('function')
transport.pauseReads?.()
transport.resumeReads?.()
expect(child.stdout.pause).toHaveBeenCalledOnce()
expect(child.stdout.resume).toHaveBeenCalledOnce()
})
it('resolves past leading garbage and hands trailing bytes to onData', async () => {
@@ -149,4 +159,25 @@ describe('waitForWslRelaySentinel', () => {
expect(err.message).toContain('E_FAIL')
expect(err.message).not.toContain(String.fromCharCode(0))
})
it('forwards write(false), callback settlement, and drain from WSL stdin', async () => {
const child = fakeChild()
const callback = vi.fn()
const drain = vi.fn()
const writeMock = child.stdin.write as ReturnType<typeof vi.fn>
writeMock.mockImplementation((_data, onWritten) => {
onWritten(null)
return false
})
const promise = waitForWslRelaySentinel(child)
emitStdout(child, RELAY_SENTINEL)
const transport = await promise
transport.onDrain?.(drain)
expect(transport.write(Buffer.from('frame'), callback)).toBe(false)
expect(callback).toHaveBeenCalledWith({ ok: true })
child.stdin.emit('drain')
expect(drain).toHaveBeenCalledOnce()
expect(transport.supportsWriteSettlement).toBe(true)
})
})
@@ -123,12 +123,15 @@ export function waitForWslRelaySentinel(
pendingChunks.push(trailing)
}
const transport: MultiplexerTransport = {
write: (data) => {
try {
child.stdin.write(data)
} catch {
// Channel already closing — mux close handling takes over.
}
write: (data, onSettled) => {
return child.stdin.write(data, (error?: Error | null) => {
onSettled?.(error ? { ok: false, error } : { ok: true })
})
},
supportsWriteSettlement: true,
onDrain: (cb) => {
child.stdin.on('drain', cb)
return () => child.stdin.off('drain', cb)
},
onData: (cb) => {
dataCallbacks.push(cb)
@@ -143,6 +146,8 @@ export function waitForWslRelaySentinel(
}
},
onClose: (cb) => closeCallbacks.push(cb),
pauseReads: () => child.stdout.pause(),
resumeReads: () => child.stdout.resume(),
close: () => child.kill()
}
resolve(transport)
@@ -9,6 +9,8 @@ export type PendingPtyData = {
droppedOutput?: true
droppedMode2031Data?: string
droppedMode2031ScanState?: Mode2031ReplyScanState
projectionAdmissionIds?: readonly string[]
projectionAdmissionsTransferred?: true
}
export type PtyPendingDataDrainDisposition = 'active' | 'background' | 'blocked'
@@ -0,0 +1,64 @@
import { describe, expect, it, vi } from 'vitest'
import {
appendPendingProjectionAdmission,
compactPendingProjectionAdmissions,
PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS,
propagatePendingProjectionRemainder
} from './pty-pending-projection-admissions'
describe('pending PTY projection admissions', () => {
it('retains the exact cap then transfers the whole ordered run on overflow', () => {
const transfer = vi.fn()
let state = compactPendingProjectionAdmissions()
for (let index = 0; index < PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS; index++) {
state = appendPendingProjectionAdmission(state, `projection-${index}`, {
isPending: () => true,
transfer
})
}
expect(state.projectionAdmissionIds).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS)
expect(transfer).not.toHaveBeenCalled()
state = appendPendingProjectionAdmission(state, 'projection-overflow', {
isPending: () => true,
transfer
})
expect(state).toEqual({ projectionAdmissionsTransferred: true })
expect(transfer).toHaveBeenCalledOnce()
expect(transfer.mock.calls[0]?.[0]).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS + 1)
})
it('compacts terminal prefixes and transfers later admissions until the remainder drains', () => {
const transfer = vi.fn()
const compacted = propagatePendingProjectionRemainder(
{
projectionAdmissionIds: ['projection-published', 'projection-partial', 'projection-tail']
},
{ sent: true, projectionsTransferred: false },
{ isPending: (id) => id !== 'projection-published', transfer }
)
expect(compacted).toEqual({
projectionAdmissionIds: ['projection-partial', 'projection-tail']
})
const transferred = appendPendingProjectionAdmission(
{ projectionAdmissionsTransferred: true },
'projection-after-transfer',
{ isPending: () => true, transfer }
)
expect(transferred).toEqual({ projectionAdmissionsTransferred: true })
expect(transfer).toHaveBeenCalledWith(['projection-after-transfer'], 'pending-projection-cap')
expect(
propagatePendingProjectionRemainder(
compacted,
{ sent: true, projectionsTransferred: true },
{ isPending: () => true, transfer }
)
).toEqual({ projectionAdmissionsTransferred: true })
})
})
@@ -0,0 +1,57 @@
export const PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS = 1024
export type PendingProjectionAdmissions = Readonly<{
projectionAdmissionIds?: readonly string[]
projectionAdmissionsTransferred?: true
}>
type PendingProjectionAdmissionOptions = Readonly<{
isPending: (id: string) => boolean
transfer: (ids: readonly string[], reason: string) => void
}>
const DEFAULT_OPTIONS: PendingProjectionAdmissionOptions = {
isPending: () => true,
transfer: () => {}
}
export function compactPendingProjectionAdmissions(
state: PendingProjectionAdmissions = {},
options: PendingProjectionAdmissionOptions = DEFAULT_OPTIONS
): PendingProjectionAdmissions {
if (state.projectionAdmissionsTransferred) {
return { projectionAdmissionsTransferred: true }
}
const ids = Array.from(
new Set((state.projectionAdmissionIds ?? []).filter((id) => options.isPending(id)))
)
return ids.length > 0 ? { projectionAdmissionIds: ids } : {}
}
export function appendPendingProjectionAdmission(
state: PendingProjectionAdmissions,
id: string,
options: PendingProjectionAdmissionOptions
): PendingProjectionAdmissions {
if (state.projectionAdmissionsTransferred) {
options.transfer([id], 'pending-projection-cap')
return { projectionAdmissionsTransferred: true }
}
const compacted = compactPendingProjectionAdmissions(state, options)
const ids = [...(compacted.projectionAdmissionIds ?? []), id]
if (ids.length <= PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS) {
return { projectionAdmissionIds: ids }
}
options.transfer(ids, 'pending-projection-cap')
return { projectionAdmissionsTransferred: true }
}
export function propagatePendingProjectionRemainder(
state: PendingProjectionAdmissions,
delivery: Readonly<{ sent: boolean; projectionsTransferred: boolean }>,
options: PendingProjectionAdmissionOptions
): PendingProjectionAdmissions {
return delivery.sent && !delivery.projectionsTransferred
? compactPendingProjectionAdmissions(state, options)
: { projectionAdmissionsTransferred: true }
}
+198 -1
View File
@@ -247,6 +247,11 @@ import { resolveWindowsShellLaunchArgs } from '../providers/windows-shell-args'
import { _resetWslCachesForTests, _setWslCachesForTests } from '../wsl'
import { wslHookRelayManager } from '../agent-hooks/wsl-hook-relay-manager'
import { acquireWatcherRemovalGate } from './watcher-removal-gate'
import {
acceptSshPtyOutputData,
acceptSshPtyOutputExit,
closeSshPtyOutputGeneration
} from './ssh-pty-output-intake-registry'
// Why: Windows resolves a bare PowerShell name to an absolute exe before ConPTY, else CreateProcessW fails with error 5 (PR #6537 / #5161).
const RESOLVED_WINDOWS_POWERSHELL = 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe'
@@ -449,7 +454,8 @@ describe('registerPtyHandlers', () => {
'ssh-reattach-1',
'ssh-reattach-fail',
'ssh-reattach-ok',
'ssh-runtime-env'
'ssh-runtime-env',
'ssh-generation-replacement'
]) {
unregisterSshPtyProvider(leakedConnectionId)
}
@@ -11556,6 +11562,197 @@ describe('registerPtyHandlers', () => {
}
})
it('keeps negotiated source-credit overflow off the legacy PTY-global pause path', async () => {
vi.useFakeTimers()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
const provider = installObservableDaemonTestProvider()
let modelSequence = 0
const runtime = {
setPtyController: vi.fn(),
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
getPtyOutputSequence: vi.fn(() => modelSequence),
onPtyData: vi.fn(
(_id: string, data: string, _at: number, rawLength = data.length) =>
(modelSequence += rawLength)
),
acceptPtyDataBounded: vi.fn(
(_id: string, _data: string, _at: number, rawLength: number) => {
modelSequence += rawLength
return { sequence: modelSequence, completion: Promise.resolve() }
}
)
}
registerPtyHandlers(mainWindow as never, runtime as never)
mainWindow.webContents.send.mockClear()
const sourceChunk = 's'.repeat(128 * 1024)
for (let index = 0; index < 17; index++) {
const sourceStartSu = index * sourceChunk.length
await acceptSshPtyOutputData({
id: 'source-credit-pty',
data: sourceChunk,
providerGeneration: 41,
ptyIncarnation: 'source-incarnation',
rawLength: sourceChunk.length,
transformed: false,
source: {
relayPtyId: 'relay-source-pty',
spanId: `source-token:${sourceStartSu}:${sourceStartSu + sourceChunk.length}`,
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'source-token',
sourceStartSu,
sourceEndSu: sourceStartSu + sourceChunk.length
}
})
}
expect(getPtyRendererDeliveryDebugSnapshot()).toMatchObject({
pendingPtyCount: 1,
pendingChars: 0
})
expect(provider.pauseProducer).not.toHaveBeenCalledWith('source-credit-pty')
expect(provider.resumeProducer).not.toHaveBeenCalledWith('source-credit-pty')
provider.emitData('legacy-pty', 'l'.repeat(320 * 1024))
expect(provider.pauseProducer).toHaveBeenCalledTimes(1)
expect(provider.pauseProducer).toHaveBeenCalledWith('legacy-pty')
expect(provider.pauseProducer).not.toHaveBeenCalledWith('unrelated-pty')
vi.runAllTimers()
expect(provider.resumeProducer).toHaveBeenCalledTimes(1)
expect(provider.resumeProducer).toHaveBeenCalledWith('legacy-pty')
} finally {
errorSpy.mockRestore()
vi.useRealTimers()
}
})
it('pauses and resumes the exact SSH provider generation across reconnect replacement', async () => {
vi.useFakeTimers()
const completion = makeDeferred()
let sequence = 0
let captures = 0
const runtime = {
setPtyController: vi.fn(),
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
getPtyOutputSequence: vi.fn(() => sequence),
acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => {
sequence += rawLength
captures++
return {
sequence,
completion: captures === 1 ? completion.promise : Promise.resolve()
}
})
}
const original = {
providerGeneration: 41,
hasPtyDeliveryPauseAdapter: () => true,
pauseProducer: vi.fn(),
resumeProducer: vi.fn()
}
const replacement = {
providerGeneration: 42,
hasPtyDeliveryPauseAdapter: () => true,
pauseProducer: vi.fn(),
resumeProducer: vi.fn()
}
const id = 'ssh:ssh-generation-replacement@@relay-pty'
const receipts: Promise<unknown>[] = []
try {
registerPtyHandlers(mainWindow as never, runtime as never)
registerSshPtyProvider('ssh-generation-replacement', original as never)
const running = acceptSshPtyOutputData({
id,
data: 'a'.repeat(256 * 1024),
providerGeneration: 41,
ptyIncarnation: 'incarnation-41',
rawLength: 256 * 1024,
transformed: false
})
receipts.push(running)
registerSshPtyProvider('ssh-generation-replacement', replacement as never)
const pressured = acceptSshPtyOutputData({
id,
data: 'b',
providerGeneration: 41,
ptyIncarnation: 'incarnation-41',
rawLength: 1,
transformed: false
})
receipts.push(pressured)
expect(original.pauseProducer).toHaveBeenCalledWith(id)
expect(replacement.pauseProducer).not.toHaveBeenCalled()
completion.resolve()
await Promise.all([running, pressured])
expect(original.resumeProducer).toHaveBeenCalledWith(id)
expect(replacement.resumeProducer).not.toHaveBeenCalled()
} finally {
completion.resolve()
await Promise.allSettled(receipts)
closeSshPtyOutputGeneration(41, 'test-cleanup')
unregisterSshPtyProvider('ssh-generation-replacement')
}
})
it('rejects local data while an SSH renderer exit waits for projection settlement', async () => {
const provider = installObservableDaemonTestProvider()
let sequence = 0
const runtime = {
setPtyController: vi.fn(),
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
getPtyOutputSequence: vi.fn(() => sequence),
acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => {
sequence += rawLength
return { sequence, completion: Promise.resolve() }
}),
onPtyData: vi.fn(),
onPtyExit: vi.fn()
}
const id = 'ssh:exit-data-race@@relay-pty'
registerPtyHandlers(mainWindow as never, runtime as never)
mainWindow.webContents.send.mockClear()
await acceptSshPtyOutputData({
id,
data: 'before-exit',
providerGeneration: 51,
ptyIncarnation: 'incarnation-51',
rawLength: 'before-exit'.length,
transformed: false
})
const exit = acceptSshPtyOutputExit({
id,
code: 0,
providerGeneration: 51,
ptyIncarnation: 'incarnation-51'
})
await Promise.resolve()
provider.emitData(id, 'must-not-follow-exit')
expect(mainWindow.webContents.send).not.toHaveBeenCalledWith('pty:data', {
id,
data: 'must-not-follow-exit'
})
getPtyAckDataListener()(null, { id, processedChars: 'before-exit'.length })
await exit
expect(mainWindow.webContents.send.mock.calls.at(-1)).toEqual([
'pty:exit',
{
id,
code: 0,
providerGeneration: 51,
ptyIncarnation: 'incarnation-51'
}
])
})
it('resumes a paused producer when the PTY exits before draining', async () => {
vi.useFakeTimers()
try {
+471 -168
View File
@@ -158,6 +158,19 @@ import {
unmarkHiddenRendererPty
} from './pty-hidden-delivery-gate'
import { PtyPendingDataDrainQueue, type PendingPtyData } from './pty-pending-data-drain-queue'
import {
appendPendingProjectionAdmission,
compactPendingProjectionAdmissions,
propagatePendingProjectionRemainder,
type PendingProjectionAdmissions
} from './pty-pending-projection-admissions'
import { SshPtyOutputIntake } from './ssh-pty-output-intake'
import {
cancelSshPtySourceDelivery,
installSshPtyOutputIntake,
publishSshPtySourceAck
} from './ssh-pty-output-intake-registry'
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
import {
clearNativeWindowsConptyPty,
isNativeWindowsLocalPtySpawn,
@@ -202,6 +215,7 @@ type FreshLocalFallbackProvider = IPtyProvider & {
routesFreshSpawnsToLocalProvider?: true
}
const sshProviders = new Map<string, IPtyProvider>()
const sshProvidersByGeneration = new Map<number, IPtyProvider>()
type RegisteredPtyProvider = {
provider: IPtyProvider
@@ -1296,10 +1310,19 @@ function beginPtySpawnForWorktree(
/** Register an SSH PTY provider for a connection. */
export function registerSshPtyProvider(connectionId: string, provider: IPtyProvider): void {
sshProviders.set(connectionId, provider)
const generation = (provider as { providerGeneration?: number }).providerGeneration
if (Number.isSafeInteger(generation) && generation! > 0) {
sshProvidersByGeneration.set(generation!, provider)
}
}
/** Remove an SSH PTY provider when a connection is closed. */
export function unregisterSshPtyProvider(connectionId: string): void {
const provider = sshProviders.get(connectionId)
const generation = (provider as { providerGeneration?: number } | undefined)?.providerGeneration
if (generation !== undefined && sshProvidersByGeneration.get(generation) === provider) {
sshProvidersByGeneration.delete(generation)
}
sshProviders.delete(connectionId)
}
@@ -1467,6 +1490,7 @@ let rendererDidStartLoadingHandler: (() => void) | null = null
// Why: Restart daemon must re-bind provider→renderer listeners after replaceDaemonProvider swaps localProvider, else subscribers stay bound to the disposed adapter and new PTY data silently drops.
let rebindProviderListeners: (() => void) | null = null
let sshOutputIntakeCleanup: (() => void) | null = null
export function rebindLocalProviderListeners(): void {
rebindProviderListeners?.()
@@ -1840,8 +1864,10 @@ export function registerPtyHandlers(
},
() => isHiddenPtyDeliveryGateEnabled(getSettings?.())
)
let sshOutputIntake: SshPtyOutputIntake | null = null
// Why: resuming a paused producer during exit can synchronously emit; those bytes must not queue behind pty:exit.
const rendererExitingPtyIds = new Set<string>()
const rendererCreditBeforeExitByPty = new Map<string, boolean>()
const rendererDeliveryRestoreNeededPtys = new Set<string>()
function transitionHiddenRendererPtyDeliveryState(id: string, hidden: boolean) {
@@ -1927,11 +1953,18 @@ export function registerPtyHandlers(
pauseProducer: (id) => tryGetProviderForPty(id)?.pauseProducer?.(id),
resumeProducer: (id) => tryGetProviderForPty(id)?.resumeProducer?.(id)
})
const sourceCreditPendingPtys = new Set<string>()
function updateProducerFlowControl(id: string): void {
if (!PRODUCER_FLOW_CONTROL_ENABLED) {
return
}
if (sourceCreditPendingPtys.has(id)) {
if (pendingData.get(id)) {
return
}
sourceCreditPendingPtys.delete(id)
}
producerFlowControl.update(id, pendingData.get(id)?.data.length ?? 0)
}
@@ -2000,7 +2033,16 @@ export function registerPtyHandlers(
}
function clearPendingPtyData(): void {
for (const pending of pendingData.values()) {
if (pending.projectionAdmissionIds) {
sshOutputIntake?.transferProjections(
pending.projectionAdmissionIds,
'renderer-lifecycle-reset'
)
}
}
pendingData.clear()
sourceCreditPendingPtys.clear()
}
function readCurrentPtyRendererDeliveryDebugSnapshot(): PtyRendererDeliveryDebugSnapshot {
@@ -2169,6 +2211,9 @@ export function registerPtyHandlers(
producerFlowControl.releaseAll()
clearDeliveryResyncProbe()
deliveryResyncUnansweredWarnLogged = false
for (const id of rendererDeliveryAccountingByPty.keys()) {
sshOutputIntake?.transferPtyProjections(id, 'renderer-lifecycle-reset')
}
rendererDeliveryAccountingByPty.clear()
rendererInFlightTotalChars = 0
clearPendingPtyData()
@@ -2265,6 +2310,9 @@ export function registerPtyHandlers(
accounting.lastAckAtMs = Date.now()
}
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - acknowledged)
if (acknowledged > 0) {
sshOutputIntake?.settleProjectionPrefix(id, acknowledged)
}
return acknowledged
}
@@ -2336,6 +2384,12 @@ export function registerPtyHandlers(
// Why drop pending: everything at/before markerSeq comes from the snapshot, so flushing pre-marker bytes would double-paint the restore.
const pending = pendingData.get(id)
if (pending) {
if (pending.projectionAdmissionIds) {
sshOutputIntake?.transferProjections(
pending.projectionAdmissionIds,
'renderer-delivery-writeoff'
)
}
pendingDroppedChars += pending.data.length
deletePendingPtyData(id)
pendingOverflowMarkedPtys.delete(id)
@@ -2365,7 +2419,11 @@ export function registerPtyHandlers(
return writtenOff
}
function sendPtyDataToRenderer(id: string, payload: PtyDataPayload): boolean {
function sendPtyDataToRenderer(
id: string,
payload: PtyDataPayload,
projectionAdmissionIds?: readonly string[]
): { sent: boolean; projectionsTransferred: boolean } {
const charCount = getPtyPayloadCharCount(payload)
const accounting = rendererDeliveryAccountingByPty.get(id)
const hadAccounting = accounting !== undefined
@@ -2400,12 +2458,28 @@ export function registerPtyHandlers(
}
}
rendererDeliveryRestoreNeededPtys.add(id)
if (projectionAdmissionIds) {
sshOutputIntake?.transferProjections(projectionAdmissionIds, 'renderer-send-failed')
}
mainDeliveryBreadcrumbs.record('pty-data-send-failed', {
id: redactPtyIdForDiagnostics(id),
chars: charCount
})
console.error('[pty] renderer data send failed; payload will not be retried', error)
return false
return { sent: false, projectionsTransferred: projectionAdmissionIds !== undefined }
}
let projectionsTransferred = false
if (projectionAdmissionIds) {
try {
sshOutputIntake?.publishProjectionPrefix(
projectionAdmissionIds,
payload.data.length,
charCount
)
} catch {
sshOutputIntake?.transferProjections(projectionAdmissionIds, 'projection-publish-failed')
projectionsTransferred = true
}
}
if (rendererDeliveryRestoreNeededPtys.has(id)) {
try {
@@ -2418,7 +2492,7 @@ export function registerPtyHandlers(
)
}
}
return true
return { sent: true, projectionsTransferred }
}
function rendererPtyIsKnownHidden(id: string): boolean {
@@ -2521,6 +2595,9 @@ export function registerPtyHandlers(
pendingOverflowMarkedPtys.add(id)
}
pendingDroppedChars += pending.data.length
if (pending.projectionAdmissionIds) {
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'pending-cap')
}
const mode2031 = scanDroppedMode2031Data(pending.data, INITIAL_MODE_2031_REPLY_SCAN_STATE)
// Why no trimmed content tail: a mid-stream gap would corrupt the pane; the droppedOutput sentinel repaints from the snapshot and realigns by sequence (only query bytes ride along).
return {
@@ -2531,6 +2608,39 @@ export function registerPtyHandlers(
}
}
function updatePendingProjectionAdmissions(
pending: PendingPtyData,
state: PendingProjectionAdmissions
): void {
delete pending.projectionAdmissionIds
delete pending.projectionAdmissionsTransferred
if (state.projectionAdmissionIds) {
pending.projectionAdmissionIds = state.projectionAdmissionIds
}
if (state.projectionAdmissionsTransferred) {
pending.projectionAdmissionsTransferred = true
}
}
function compactPendingProjectionState(
pending: PendingProjectionAdmissions,
projectionSemanticsId?: string
): PendingProjectionAdmissions {
const options = pendingProjectionAdmissionOptions()
const compacted = compactPendingProjectionAdmissions(pending, options)
return projectionSemanticsId
? appendPendingProjectionAdmission(compacted, projectionSemanticsId, options)
: compacted
}
function pendingProjectionAdmissionOptions() {
return {
isPending: (id: string) => sshOutputIntake?.hasUnpublishedProjection(id) ?? false,
transfer: (ids: readonly string[], reason: string) =>
sshOutputIntake?.transferProjections(ids, reason)
}
}
function appendPendingPtyData(
id: string,
existing: PendingPtyData | undefined,
@@ -2539,10 +2649,14 @@ export function registerPtyHandlers(
preservesSeq: boolean,
containsBackgroundOutput: boolean,
rawLength = data.length,
transformed = false
transformed = false,
projectionSemanticsId?: string
): PendingPtyData {
// Why stay dropped at O(1): once over the cap the restore sentinel supersedes interim bytes; queries still get carved out (bounded) so replies survive the whole episode.
if (existing?.droppedOutput === true) {
if (projectionSemanticsId) {
sshOutputIntake?.transferProjections([projectionSemanticsId], 'pending-cap')
}
const mode2031 = scanDroppedMode2031Data(
data,
existing.droppedMode2031ScanState ?? INITIAL_MODE_2031_REPLY_SCAN_STATE
@@ -2559,16 +2673,19 @@ export function registerPtyHandlers(
droppedMode2031ScanState: mode2031.state
}
}
const projectionState = compactPendingProjectionState(existing ?? {}, projectionSemanticsId)
const nextContainsBackgroundOutput =
existing?.containsBackgroundOutput === true || containsBackgroundOutput
if (!existing) {
return dropOversizedPendingPtyData(id, {
const pending: PendingPtyData = {
data,
...(typeof startSeq === 'number' ? { startSeq } : {}),
...(rawLength !== data.length ? { rawLength } : {}),
...(transformed ? { transformed: true } : {}),
...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {})
})
}
updatePendingProjectionAdmissions(pending, projectionState)
return dropOversizedPendingPtyData(id, pending)
}
const existingRawLength = existing.rawLength ?? existing.data.length
const next: PendingPtyData = {
@@ -2578,6 +2695,7 @@ export function registerPtyHandlers(
: {}),
...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {})
}
updatePendingProjectionAdmissions(next, projectionState)
if (typeof existing.startSeq === 'number') {
next.startSeq = existing.startSeq
}
@@ -2661,6 +2779,9 @@ export function registerPtyHandlers(
pendingOverflowMarkedPtys.delete(id)
updateProducerFlowControl(id)
const drop = recordHiddenRendererPtyDataDrop(id, pending.data.length)
if (pending.projectionAdmissionIds) {
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop')
}
warnIfDroppingHiddenBytesForVisiblePty(id, pending.data.length)
if (drop.shouldEmitRestoreMarker) {
sendModelRestoreNeededMarker(id, 'hidden-drop', runtime?.getPtyOutputSequence(id))
@@ -2676,11 +2797,15 @@ export function registerPtyHandlers(
updateProducerFlowControl(id)
// Why droppedOutput sentinel: pending-cap drop means the pane must repaint from the snapshot, not continue a gapped stream (data = carved query bytes only).
if (
!sendPtyDataToRenderer(id, {
!sendPtyDataToRenderer(
id,
data: pending.data + getDroppedMode2031RendererData(pending),
droppedOutput: true
})
{
id,
data: pending.data + getDroppedMode2031RendererData(pending),
droppedOutput: true
},
pending.projectionAdmissionIds
).sent
) {
sendFailed = true
break
@@ -2692,33 +2817,50 @@ export function registerPtyHandlers(
const indivisible = pending.transformed === true
const chunk = indivisible ? data : data.slice(0, PTY_BATCH_FLUSH_CHUNK_CHARS)
const remaining = indivisible ? '' : data.slice(PTY_BATCH_FLUSH_CHUNK_CHARS)
let nextPending: PendingPtyData | undefined
if (remaining) {
const nextPending: PendingPtyData = { data: remaining }
nextPending = { data: remaining }
if (typeof pending.startSeq === 'number') {
nextPending.startSeq = pending.startSeq + chunk.length
}
if (pending.containsBackgroundOutput === true) {
nextPending.containsBackgroundOutput = true
}
if (pending.projectionAdmissionIds) {
nextPending.projectionAdmissionIds = pending.projectionAdmissionIds
}
if (pending.projectionAdmissionsTransferred) {
nextPending.projectionAdmissionsTransferred = true
}
pendingData.replaceWithRemainder(selection, nextPending)
} else {
pendingData.remove(selection)
pendingOverflowMarkedPtys.delete(id)
}
updateProducerFlowControl(id)
if (
!sendPtyDataToRenderer(
const delivery = sendPtyDataToRenderer(
id,
makePtyDataPayload(
id,
makePtyDataPayload(
id,
chunk,
pending.startSeq,
pending.containsBackgroundOutput,
pending.rawLength,
pending.transformed
chunk,
pending.startSeq,
pending.containsBackgroundOutput,
pending.rawLength,
pending.transformed
),
pending.projectionAdmissionIds
)
if (nextPending) {
updatePendingProjectionAdmissions(
nextPending,
propagatePendingProjectionRemainder(
nextPending,
delivery,
pendingProjectionAdmissionOptions()
)
)
) {
}
if (!delivery.sent) {
sendFailed = true
break
}
@@ -2781,27 +2923,45 @@ export function registerPtyHandlers(
return true
}
function sendPtyExitToRenderer(payload: { id: string; code: number }): void {
function preparePtyExitForRenderer(payload: { id: string; code: number }): (() => void) | null {
if (mainWindow.isDestroyed()) {
return
sshOutputIntake?.transferPtyProjections(payload.id, 'renderer-destroyed')
return () => {}
}
if (rendererExitingPtyIds.has(payload.id)) {
return
return null
}
rendererExitingPtyIds.add(payload.id)
let released = false
const release = (): void => {
if (released) {
return
}
released = true
rendererExitingPtyIds.delete(payload.id)
}
try {
const hadReleasableRendererCredit = getRendererInFlightCharsForPty(payload.id) > 0
if (!rendererCreditBeforeExitByPty.has(payload.id)) {
rendererCreditBeforeExitByPty.set(
payload.id,
getRendererInFlightCharsForPty(payload.id) > 0
)
}
// Why flush before exit: the renderer tears down the terminal on pty:exit, so any batched output not yet flushed would be silently lost.
const remaining = pendingData.delete(payload.id)
clearFlushTimerIfIdle()
if (remaining) {
if (remaining.droppedOutput === true) {
// Sentinel entry: only salvaged query bytes remain; keep the flag so the renderer knows the span was dropped.
sendPtyDataToRenderer(payload.id, {
id: payload.id,
data: remaining.data,
droppedOutput: true
})
sendPtyDataToRenderer(
payload.id,
{
id: payload.id,
data: remaining.data,
droppedOutput: true
},
remaining.projectionAdmissionIds
)
} else {
sendPtyDataToRenderer(
payload.id,
@@ -2812,35 +2972,63 @@ export function registerPtyHandlers(
remaining.containsBackgroundOutput,
remaining.rawLength,
remaining.transformed
)
),
remaining.projectionAdmissionIds
)
}
}
// Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id.
producerFlowControl.release(payload.id)
pendingOverflowMarkedPtys.delete(payload.id)
rendererDeliveryRestoreNeededPtys.delete(payload.id)
lastInputAtByPty.delete(payload.id)
interactiveOutputCharsByPty.delete(payload.id)
const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id)
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit)
// Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides.
rendererDeliveryAccountingByPty.delete(payload.id)
if (hadReleasableRendererCredit) {
if (pendingDataFlushActive) {
// Why: let the open round coalesce this wake into its one post-round continuation.
const reactivatedBlocked = pendingData.reactivateBlocked()
pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked
} else {
schedulePendingDataAfterCreditReport(true)
}
return release
} catch (error) {
release()
throw error
}
}
function finalizePtyExitForRenderer(payload: { id: string; code: number }): void {
if (mainWindow.isDestroyed()) {
rendererCreditBeforeExitByPty.delete(payload.id)
return
}
const hadReleasableRendererCredit =
rendererCreditBeforeExitByPty.get(payload.id) ??
getRendererInFlightCharsForPty(payload.id) > 0
rendererCreditBeforeExitByPty.delete(payload.id)
// Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id.
producerFlowControl.release(payload.id)
sourceCreditPendingPtys.delete(payload.id)
pendingOverflowMarkedPtys.delete(payload.id)
rendererDeliveryRestoreNeededPtys.delete(payload.id)
lastInputAtByPty.delete(payload.id)
interactiveOutputCharsByPty.delete(payload.id)
const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id)
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit)
// Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides.
rendererDeliveryAccountingByPty.delete(payload.id)
if (hadReleasableRendererCredit) {
if (pendingDataFlushActive) {
// Why: let the open round coalesce this wake into its one post-round continuation.
const reactivatedBlocked = pendingData.reactivateBlocked()
pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked
} else {
schedulePendingDataAfterCreditReport(true)
}
mainWindow.webContents.send('pty:exit', {
...payload,
...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {})
})
}
mainWindow.webContents.send('pty:exit', {
...payload,
...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {})
})
}
function sendPtyExitToRenderer(payload: { id: string; code: number }): void {
const release = preparePtyExitForRenderer(payload)
if (!release) {
return
}
try {
sshOutputIntake?.transferPtyProjections(payload.id, 'legacy-pty-exit')
finalizePtyExitForRenderer(payload)
} finally {
rendererExitingPtyIds.delete(payload.id)
release()
}
}
@@ -2850,6 +3038,231 @@ export function registerPtyHandlers(
}
}
function acceptPtyDataForRenderer(
payload: {
id: string
data: string
sequenceChars?: number
transformed?: boolean
},
outputSeq: number | undefined,
projection?: LegacySshProjectionSemantics
): void {
const rawLength = payload.sequenceChars ?? payload.data.length
const preservesSeq = !payload.transformed && rawLength === payload.data.length
const startSeq = typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined
const projectionId = projection?.identity.projectionSemanticsId
if (mainWindow.isDestroyed()) {
if (projectionId) {
sshOutputIntake?.transferProjections([projectionId], 'renderer-destroyed')
}
if (flushTimer) {
clearTimeout(flushTimer)
flushTimer = null
}
producerFlowControl.releaseAll()
clearDeliveryResyncProbe()
clearPendingPtyData()
pendingOverflowMarkedPtys.clear()
rendererDeliveryAccountingByPty.clear()
rendererInFlightTotalChars = 0
clearDispatcherReadyWatchdog()
return
}
if (rendererExitingPtyIds.has(payload.id)) {
if (projectionId) {
sshOutputIntake?.transferProjections([projectionId], 'pty-exiting')
}
return
}
if (shouldDropHiddenRendererPtyData(payload.id, getSettings?.())) {
if (projectionId) {
sshOutputIntake?.transferProjections([projectionId], 'hidden-drop')
}
const droppedChars = projection ? rawLength : payload.data.length
const drop = recordHiddenRendererPtyDataDrop(payload.id, droppedChars)
warnIfDroppingHiddenBytesForVisiblePty(payload.id, droppedChars)
if (drop.shouldEmitRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq)
}
return
}
if (payload.data.length === 0 && !payload.transformed) {
if (projectionId) {
sshOutputIntake?.transferProjections([projectionId], 'empty-projection')
}
return
}
const containsBackgroundOutput =
rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id)
if (containsBackgroundOutput) {
markHiddenRendererResizeOutputDelivered(payload.id)
}
const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id)
if (projection?.desktopSpan) {
sourceCreditPendingPtys.add(payload.id)
}
const pending = appendPendingPtyData(
payload.id,
pendingData.get(payload.id),
payload.data,
startSeq,
preservesSeq,
containsBackgroundOutput,
rawLength,
payload.transformed === true,
projectionId
)
const shouldEmitPendingCapRestoreMarker =
pending.droppedOutput === true &&
!overflowMarkedBeforeAppend &&
pendingOverflowMarkedPtys.has(payload.id)
const nextData = pending.data + getDroppedMode2031RendererData(pending)
const isInteractiveOutput = shouldSendInteractiveOutputNow(
payload.id,
nextData,
performance.now()
)
if (isInteractiveOutput && rendererPtyDispatcherReady) {
if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) {
setPendingPtyData(payload.id, pending)
if (shouldEmitPendingCapRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
}
updateProducerFlowControl(payload.id)
requestDeliveryResyncForGatedPty()
return
}
deletePendingPtyData(payload.id)
clearFlushTimerIfIdle()
if (shouldEmitPendingCapRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
}
pendingOverflowMarkedPtys.delete(payload.id)
try {
sendPtyDataToRenderer(
payload.id,
{
id: payload.id,
data: nextData,
...(typeof pending.startSeq === 'number'
? {
seq: pending.startSeq + (pending.rawLength ?? nextData.length),
rawLength: pending.rawLength ?? nextData.length
}
: {}),
...(pending.transformed ? { transformed: true } : {}),
...(pending.containsBackgroundOutput === true ? { background: true } : {}),
...(pending.droppedOutput === true ? { droppedOutput: true } : {})
},
pending.projectionAdmissionIds
)
} finally {
updateProducerFlowControl(payload.id)
}
return
}
setPendingPtyData(payload.id, pending)
if (shouldEmitPendingCapRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
}
updateProducerFlowControl(payload.id)
if (
!canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) })
) {
requestDeliveryResyncForGatedPty()
}
if (!flushTimer) {
schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS)
}
}
sshOutputIntakeCleanup?.()
sshOutputIntake = new SshPtyOutputIntake({
getModelSequence: (id) => runtime?.getPtyOutputSequence(id) ?? 0,
acceptModel: (event, projection) => {
if (!runtime) {
throw new Error('SSH PTY output requires the main terminal model')
}
return runtime.acceptPtyDataBounded(
event.id,
event.data,
Date.now(),
event.rawLength,
event.transformed,
projection.desktopSpan ? [projection.desktopSpan] : undefined
)
},
project: (event, projection) =>
acceptPtyDataForRenderer(
{
id: event.id,
data: event.data,
sequenceChars: event.rawLength,
transformed: event.transformed
},
projection.identity.sequenceEnd,
projection
),
prepareExit: (event) => {
const release = preparePtyExitForRenderer(event)
if (!release) {
throw new Error('pty_renderer_exit_in_progress')
}
return release
},
finalizeExit: (event) => {
runtime?.onPtyExit(event.id, event.code, event.ptyIncarnation)
finalizePtyExitForRenderer(event)
},
pauseProvider: (generation, id) => {
const provider = sshProvidersByGeneration.get(generation) as
| (IPtyProvider & { hasPtyDeliveryPauseAdapter?: () => boolean })
| undefined
if (!provider?.hasPtyDeliveryPauseAdapter?.()) {
return false
}
provider.pauseProducer?.(id)
return true
},
resumeProvider: (generation, id) =>
sshProvidersByGeneration.get(generation)?.resumeProducer?.(id),
closeProvider: (generation, reason) => {
const provider = sshProvidersByGeneration.get(generation)
;(
provider as (IPtyProvider & { closeOutputIntake?: (reason: string) => void }) | undefined
)?.closeOutputIntake?.(reason)
},
resetModelForMigration: (_generation, id) => runtime?.resetPtyModelAfterMigrationFailure(id),
onGenerationClosed: (providerGeneration) => {
for (const id of pendingData.keys()) {
const pending = pendingData.get(id)
if (
pending?.projectionAdmissionIds &&
sshOutputIntake?.hasProjectionFromGeneration(
pending.projectionAdmissionIds,
providerGeneration
)
) {
pendingData.delete(id)
updateProducerFlowControl(id)
pendingOverflowMarkedPtys.delete(id)
}
}
sshProvidersByGeneration.delete(providerGeneration)
},
publishSourceAck: publishSshPtySourceAck,
cancelSourceDelivery: cancelSshPtySourceDelivery
})
runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(
sshOutputIntake.getRemoteSourceRangeConsumerHooks()
)
const cleanupSshOutputIntakeRegistry = installSshPtyOutputIntake(sshOutputIntake)
sshOutputIntakeCleanup = () => {
runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(null)
cleanupSshOutputIntakeRegistry()
}
async function shutdownProviderAndDetectExit(
provider: IPtyProvider,
id: string,
@@ -2933,120 +3346,7 @@ export function registerPtyHandlers(
const outputSeq = isLocalProvider
? runtime?.getPtyOutputSequence(payload.id)
: runtime?.onPtyData(payload.id, payload.data, Date.now(), rawLength, payload.transformed)
const rendererData = payload.data
const preservesSeq = !payload.transformed && rawLength === payload.data.length
const startSeq =
typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined
if (mainWindow.isDestroyed()) {
// Why clear the flush timer: macOS app re-activation otherwise leaks orphaned timers from the previous window's registration.
if (flushTimer) {
clearTimeout(flushTimer)
flushTimer = null
}
producerFlowControl.releaseAll()
clearDeliveryResyncProbe()
clearPendingPtyData()
pendingOverflowMarkedPtys.clear()
rendererDeliveryAccountingByPty.clear()
rendererInFlightTotalChars = 0
clearDispatcherReadyWatchdog()
return
}
if (rendererExitingPtyIds.size > 0 && rendererExitingPtyIds.has(payload.id)) {
return
}
const settings = getSettings?.()
// Why drop before the interactive bypass: runtime already ingested the chunk, so gated PTYs skip both renderer paths and reveal restores from the snapshot.
if (shouldDropHiddenRendererPtyData(payload.id, settings)) {
const drop = recordHiddenRendererPtyDataDrop(payload.id, payload.data.length)
warnIfDroppingHiddenBytesForVisiblePty(payload.id, payload.data.length)
if (drop.shouldEmitRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq)
}
return
}
if (rendererData.length === 0 && !payload.transformed) {
return
}
const containsBackgroundOutput =
rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id)
if (containsBackgroundOutput) {
markHiddenRendererResizeOutputDelivered(payload.id)
}
const existing = pendingData.get(payload.id)
const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id)
const pending = appendPendingPtyData(
payload.id,
existing,
rendererData,
startSeq,
preservesSeq,
containsBackgroundOutput,
rawLength,
payload.transformed === true
)
const shouldEmitPendingCapRestoreMarker =
pending.droppedOutput === true &&
!overflowMarkedBeforeAppend &&
pendingOverflowMarkedPtys.has(payload.id)
const nextData = pending.data + getDroppedMode2031RendererData(pending)
const isInteractiveOutput = shouldSendInteractiveOutputNow(
payload.id,
nextData,
performance.now()
)
// Why gate the fast path on the handshake too: else boot-window keystroke echo is sent into a listener-less page and pins the gate.
if (isInteractiveOutput && rendererPtyDispatcherReady) {
// Why the reserve: keep input echo from being pinned behind unrelated bulk output; it's bounded and the per-PTY cap still prevents an active TUI runaway.
if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) {
setPendingPtyData(payload.id, pending)
if (shouldEmitPendingCapRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
}
updateProducerFlowControl(payload.id)
requestDeliveryResyncForGatedPty()
return
}
deletePendingPtyData(payload.id)
clearFlushTimerIfIdle()
if (shouldEmitPendingCapRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
}
pendingOverflowMarkedPtys.delete(payload.id)
// Why immediate: agent TUIs redraw small prompt regions per keystroke; the throughput batch timer would add visible input latency.
try {
sendPtyDataToRenderer(payload.id, {
id: payload.id,
data: nextData,
...(typeof pending.startSeq === 'number'
? {
seq: pending.startSeq + (pending.rawLength ?? nextData.length),
rawLength: pending.rawLength ?? nextData.length
}
: {}),
...(pending.transformed ? { transformed: true } : {}),
...(pending.containsBackgroundOutput === true ? { background: true } : {}),
...(pending.droppedOutput === true ? { droppedOutput: true } : {})
})
} finally {
updateProducerFlowControl(payload.id)
}
return
}
setPendingPtyData(payload.id, pending)
if (shouldEmitPendingCapRestoreMarker) {
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
}
updateProducerFlowControl(payload.id)
// Why probe on data arrival (not flush skips): new output for a fully gated PTY is the moment stuck delivery becomes observable.
if (
!canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) })
) {
requestDeliveryResyncForGatedPty()
}
if (!flushTimer) {
schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS)
}
acceptPtyDataForRenderer(payload, outputSeq)
})
localExitUnsub = localProvider.onExit((payload) => {
if (!isCurrentPtyExit(payload)) {
@@ -5646,6 +5946,9 @@ export function registerPtyHandlers(
const pending = pendingData.get(args.id)
if (pending && transition.droppable) {
pendingData.delete(args.id)
if (pending.projectionAdmissionIds) {
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop')
}
updateProducerFlowControl(args.id)
pendingOverflowMarkedPtys.delete(args.id)
const drop = recordHiddenRendererPtyDataDrop(args.id, pending.data.length)
@@ -0,0 +1,53 @@
type ClosedGenerationRange = {
start: number
end: number
}
export class SshPtyClosedGenerationRanges {
private readonly ranges: ClosedGenerationRange[] = []
add(generation: number): void {
let index = 0
while (index < this.ranges.length && this.ranges[index]!.end + 1 < generation) {
index++
}
const current = this.ranges[index]
if (!current || generation + 1 < current.start) {
this.ranges.splice(index, 0, { start: generation, end: generation })
return
}
current.start = Math.min(current.start, generation)
current.end = Math.max(current.end, generation)
const next = this.ranges[index + 1]
if (next && current.end + 1 >= next.start) {
current.end = Math.max(current.end, next.end)
this.ranges.splice(index + 1, 1)
}
}
has(generation: number): boolean {
for (const range of this.ranges) {
if (generation < range.start) {
return false
}
if (generation <= range.end) {
return true
}
}
return false
}
get size(): number {
return this.ranges.length
}
get activeGaps(): number {
const highWater = this.ranges.at(-1)?.end ?? 0
let closedGenerations = 0
for (const range of this.ranges) {
closedGenerations += range.end - range.start + 1
}
// Why: provider generations allocate from 1, so unclosed IDs below high-water remain active.
return highWater - closedGenerations
}
}
@@ -0,0 +1,115 @@
import {
projectionError,
reclaimProjectionRecord,
type DesktopProjectionSpan,
type ProjectionRecord
} from './ssh-pty-legacy-projection-record'
export function publishLegacyProjectionPrefix(
records: Map<string, ProjectionRecord>,
ids: readonly string[],
displayChars: number,
accountingChars: number
): void {
let displayRemaining = Math.max(0, displayChars)
let accountingRemaining = Math.max(0, accountingChars)
for (const id of ids) {
const record = records.get(id)
if (!record) {
continue
}
const displayLength =
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
const unpublishedDisplay = displayLength - record.publishedDisplay
const unpublishedAccounting = record.semantics.identity.rawLength - record.publishedAccounting
if (displayLength === 0 && unpublishedAccounting > 0) {
const publishAccounting = Math.min(accountingRemaining, unpublishedAccounting)
if (publishAccounting !== unpublishedAccounting) {
throw projectionError('ssh_projection_indivisible_split')
}
record.publishedAccounting += publishAccounting
record.state = 'published'
accountingRemaining -= publishAccounting
continue
}
if (unpublishedDisplay <= 0) {
continue
}
const publishDisplay = Math.min(displayRemaining, unpublishedDisplay)
if (publishDisplay <= 0) {
break
}
if (record.semantics.identity.transformed && publishDisplay !== unpublishedDisplay) {
throw projectionError('ssh_projection_indivisible_split')
}
const publishAccounting =
publishDisplay === unpublishedDisplay
? Math.min(accountingRemaining, unpublishedAccounting)
: publishDisplay
record.publishedDisplay += publishDisplay
record.publishedAccounting += publishAccounting
record.state = 'published'
displayRemaining -= publishDisplay
accountingRemaining -= publishAccounting
}
if (displayRemaining !== 0 || accountingRemaining !== 0) {
throw projectionError('ssh_projection_publish_range_mismatch')
}
}
export function hasUnpublishedLegacyProjection(
records: ReadonlyMap<string, ProjectionRecord>,
id: string
): boolean {
const record = records.get(id)
if (!record || record.state === 'reserved') {
return false
}
const displayLength =
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
return (
record.publishedDisplay < displayLength ||
record.publishedAccounting < record.semantics.identity.rawLength
)
}
export function settlePublishedLegacyProjectionPrefix(
records: Map<string, ProjectionRecord>,
idsByPty: Map<string, string[]>,
ptyId: string,
accountingChars: number,
onSettled: ((span: DesktopProjectionSpan, reason: string) => void) | undefined
): { settled: number; completed: number } {
let remaining = Math.max(0, accountingChars)
let settled = 0
let completed = 0
for (const id of idsByPty.get(ptyId)?.slice() ?? []) {
const record = records.get(id)
if (!record) {
continue
}
const available = record.publishedAccounting - record.settledAccounting
if (available <= 0) {
continue
}
const take = Math.min(remaining, available)
const finishes =
record.settledAccounting + take === record.semantics.identity.rawLength &&
record.publishedDisplay ===
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
if (finishes && record.semantics.desktopSpan) {
onSettled?.(record.semantics.desktopSpan, 'renderer-parse')
}
record.settledAccounting += take
settled += take
remaining -= take
if (finishes) {
completed++
reclaimProjectionRecord(records, idsByPty, id, ptyId)
}
if (remaining === 0) {
break
}
}
return { settled, completed }
}
@@ -0,0 +1,218 @@
import type {
Mode2031ReplyDecision,
Mode2031ReplyScanState
} from '../../shared/terminal-color-scheme-protocol'
import { INITIAL_MODE_2031_REPLY_SCAN_STATE } from '../../shared/terminal-color-scheme-protocol'
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
export type LegacySshProjectionIdentity = Readonly<{
projectionSemanticsId: string
ptyId: string
providerGeneration: number
ptyIncarnation: string
displayStart: number
displayEnd: number
sequenceEnd: number
rawLength: number
transformed: boolean
}>
export type LegacySshProjectionSemantics = Readonly<{
identity: LegacySshProjectionIdentity
desktopSpan?: DesktopProjectionSpan
beforeScanner: Readonly<Mode2031ReplyScanState>
afterScanner: Readonly<Mode2031ReplyScanState>
decision: Mode2031ReplyDecision
}>
export type DesktopProjectionSpan = Readonly<
TerminalOutputSourceRange & {
projectionSemanticsId: string
transform: Readonly<{
transformed: boolean
rawLengthSu: number
scalarSafe: boolean
}>
}
>
export type ProjectionState = 'reserved' | 'committed' | 'published'
export type ProjectionRecord = {
semantics: LegacySshProjectionSemantics
state: ProjectionState
publishedDisplay: number
publishedAccounting: number
settledAccounting: number
}
export type PtyProjectionCursor = {
providerGeneration: number
ptyIncarnation: string
displayEnd: number
scanner: Mode2031ReplyScanState
}
export type LegacySshProjectionReservation = Readonly<{
semantics: LegacySshProjectionSemantics
}>
export type LegacySshProjectionDebugSnapshot = {
reserved: number
committed: number
published: number
settled: number
transferred: number
rolledBack: number
records: number
cursors: number
}
export function scannerSnapshot(state: Mode2031ReplyScanState): Readonly<Mode2031ReplyScanState> {
return Object.freeze({ tail: state.tail, pendingSubscribe: state.pendingSubscribe })
}
export function resetProjectionCursorForGap(
cursors: ReadonlyMap<string, PtyProjectionCursor>,
ptyId: string
): void {
const cursor = cursors.get(ptyId)
if (cursor) {
cursor.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
}
}
export function projectionError(code: string): Error {
return Object.assign(new Error(code), { code })
}
export function projectionDebugSnapshot(
records: ReadonlyMap<string, ProjectionRecord>,
cursors: ReadonlyMap<string, PtyProjectionCursor>,
terminalCounts: { settled: number; transferred: number; rolledBack: number }
): LegacySshProjectionDebugSnapshot {
const result = {
reserved: 0,
committed: 0,
published: 0,
...terminalCounts,
records: records.size,
cursors: cursors.size
}
for (const record of records.values()) {
result[record.state]++
}
return result
}
export function reclaimProjectionRecord(
records: Map<string, ProjectionRecord>,
idsByPty: Map<string, string[]>,
id: string,
ptyId: string
): void {
records.delete(id)
const ids = idsByPty.get(ptyId)
if (!ids) {
return
}
const index = ids.indexOf(id)
if (index >= 0) {
ids.splice(index, 1)
}
if (ids.length === 0) {
idsByPty.delete(ptyId)
}
}
export function requireProjectionRecord(
records: ReadonlyMap<string, ProjectionRecord>,
id: string
): ProjectionRecord {
const record = records.get(id)
if (!record) {
throw projectionError('ssh_projection_reservation_missing')
}
return record
}
export function rollbackCommittedProjectionRecord(
records: Map<string, ProjectionRecord>,
idsByPty: Map<string, string[]>,
cursors: Map<string, PtyProjectionCursor>,
reservation: LegacySshProjectionReservation
): string | null {
const id = reservation.semantics.identity.projectionSemanticsId
const record = records.get(id)
if (!record || record.state !== 'committed') {
return null
}
const { identity, beforeScanner } = record.semantics
const cursor = cursors.get(identity.ptyId)
if (
idsByPty.get(identity.ptyId)?.at(-1) !== id ||
!cursor ||
cursor.providerGeneration !== identity.providerGeneration ||
cursor.ptyIncarnation !== identity.ptyIncarnation ||
cursor.displayEnd !== identity.displayEnd
) {
return null
}
cursor.displayEnd = identity.displayStart
cursor.scanner = { ...beforeScanner }
reclaimProjectionRecord(records, idsByPty, id, identity.ptyId)
return identity.ptyId
}
export function closeProjectionPty(
cursors: Map<string, PtyProjectionCursor>,
ptyId: string,
providerGeneration: number,
ptyIncarnation: string,
beforeDelete: () => void
): void {
const cursor = cursors.get(ptyId)
if (
!cursor ||
cursor.providerGeneration !== providerGeneration ||
cursor.ptyIncarnation !== ptyIncarnation
) {
return
}
beforeDelete()
cursors.delete(ptyId)
}
export function getOrCreateProjectionCursor(
cursors: Map<string, PtyProjectionCursor>,
args: { ptyId: string; providerGeneration: number; ptyIncarnation: string },
replaceGeneration: (providerGeneration: number) => void
): PtyProjectionCursor {
const existing = cursors.get(args.ptyId)
if (!existing) {
const cursor = {
providerGeneration: args.providerGeneration,
ptyIncarnation: args.ptyIncarnation,
displayEnd: 0,
scanner: { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
}
cursors.set(args.ptyId, cursor)
return cursor
}
if (args.providerGeneration < existing.providerGeneration) {
throw projectionError('ssh_projection_stale_generation')
}
if (
args.providerGeneration === existing.providerGeneration &&
args.ptyIncarnation !== existing.ptyIncarnation
) {
throw projectionError('ssh_projection_stale_incarnation')
}
if (args.providerGeneration > existing.providerGeneration) {
replaceGeneration(existing.providerGeneration)
existing.providerGeneration = args.providerGeneration
existing.ptyIncarnation = args.ptyIncarnation
existing.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
}
return existing
}
@@ -0,0 +1,244 @@
import { describe, expect, it } from 'vitest'
import { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
import { SshPtyProjectionTerminality } from './ssh-pty-projection-terminality'
function reserve(
ledger: SshPtyLegacyProjectionLedger,
overrides: Partial<Parameters<SshPtyLegacyProjectionLedger['reserve']>[0]> = {}
) {
return ledger.reserve({
ptyId: 'pty-1',
providerGeneration: 3,
ptyIncarnation: 'incarnation-1',
data: 'abc',
sequenceEnd: 3,
rawLength: 3,
transformed: false,
...overrides
})
}
function source(sourceStartSu: number, sourceEndSu: number, deliveryToken = 'token-1') {
return {
spanId: `span-${sourceStartSu}`,
clientGeneration: 2,
ownerGeneration: 4,
deliveryToken,
sourceStartSu,
sourceEndSu
}
}
describe('SshPtyLegacyProjectionLedger', () => {
it('drains exact terminality waiters when their provider generation closes', async () => {
const terminality = new SshPtyProjectionTerminality()
const terminal = terminality.whenTerminal('pty-1', 3, 'incarnation-1', () => true)
let nextResolved = false
const next = terminality
.whenTerminal('pty-1', 4, 'incarnation-2', () => true)
.then(() => {
nextResolved = true
})
terminality.closeGeneration(3)
await expect(terminal).resolves.toBeUndefined()
expect(nextResolved).toBe(false)
terminality.closeGeneration(4)
await next
})
it('rolls back scanner and display reservations before commit', () => {
const ledger = new SshPtyLegacyProjectionLedger()
const partial = reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 })
expect(ledger.rollback(partial)).toBe(true)
const next = reserve(ledger, { data: '31h', sequenceEnd: 3 })
expect(next.semantics.identity.displayStart).toBe(0)
expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
expect(next.semantics.decision).toBeNull()
})
it('keeps immutable generation, incarnation, display, sequence, raw length, and scanner facts', () => {
const ledger = new SshPtyLegacyProjectionLedger()
const first = reserve(ledger, {
data: '\x1b[?2031h',
rawLength: 11,
sequenceEnd: 11
})
const semantics = ledger.commit(first)
expect(semantics.identity).toMatchObject({
providerGeneration: 3,
ptyIncarnation: 'incarnation-1',
displayStart: 0,
displayEnd: 8,
sequenceEnd: 11,
rawLength: 11
})
expect(semantics.decision).toBe('subscribed')
expect(Object.isFrozen(semantics)).toBe(true)
expect(Object.isFrozen(semantics.identity)).toBe(true)
})
it('rejects stale generations and resets cross-chunk scanner state on gaps', () => {
const ledger = new SshPtyLegacyProjectionLedger()
ledger.commit(reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 }))
ledger.resetForGap('pty-1')
const next = reserve(ledger, { data: '31h', sequenceEnd: 8 })
expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
expect(() => reserve(ledger, { providerGeneration: 2 })).toThrow(
'ssh_projection_stale_generation'
)
})
it('publishes, settles, and transfers explicit ranges', () => {
const ledger = new SshPtyLegacyProjectionLedger()
const first = ledger.commit(reserve(ledger))
ledger.publishPrefix([first.identity.projectionSemanticsId], 3, 3)
expect(ledger.settlePublishedPrefix('pty-1', 2)).toBe(2)
expect(ledger.transfer([first.identity.projectionSemanticsId], 'renderer-reload')).toBe(1)
expect(ledger.getDebugSnapshot()).toMatchObject({ transferred: 1, records: 0 })
})
it('resolves exact PTY terminal waiters only after settlement or transfer', async () => {
const ledger = new SshPtyLegacyProjectionLedger()
const projection = ledger.commit(reserve(ledger))
ledger.publishPrefix([projection.identity.projectionSemanticsId], 3, 3)
let settled = false
const terminal = ledger.whenPtyTerminal('pty-1', 3, 'incarnation-1').then(() => {
settled = true
})
await Promise.resolve()
expect(settled).toBe(false)
ledger.settlePublishedPrefix('pty-1', 2)
await Promise.resolve()
expect(settled).toBe(false)
ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload')
await terminal
expect(settled).toBe(true)
})
it('publishes and settles transformed source accounting with no display text', () => {
const ledger = new SshPtyLegacyProjectionLedger()
const projection = ledger.commit(
reserve(ledger, {
data: '',
sequenceEnd: 9,
rawLength: 9,
transformed: true
})
)
ledger.publishPrefix([projection.identity.projectionSemanticsId], 0, 9)
expect(ledger.settlePublishedPrefix('pty-1', 9)).toBe(9)
expect(ledger.getDebugSnapshot()).toMatchObject({ settled: 1, records: 0 })
})
it('reclaims a closed PTY cursor so its id can be reused by a new incarnation', () => {
const ledger = new SshPtyLegacyProjectionLedger()
ledger.commit(reserve(ledger))
ledger.closePty('pty-1', 3, 'incarnation-1', 'pty-exit')
const next = reserve(ledger, {
ptyIncarnation: 'incarnation-2',
data: 'next',
sequenceEnd: 4,
rawLength: 4
})
expect(next.semantics.identity).toMatchObject({
ptyIncarnation: 'incarnation-2',
displayStart: 0
})
expect(ledger.getDebugSnapshot()).toMatchObject({ records: 1, cursors: 1 })
})
it('keeps split publication attached to one immutable desktop span', () => {
const settled: unknown[] = []
const ledger = new SshPtyLegacyProjectionLedger({
onSettled: (span) => settled.push(span)
})
const projection = ledger.commit(
reserve(ledger, {
data: 'abcd',
rawLength: 4,
sequenceEnd: 4,
source: source(0, 4)
})
)
const id = projection.identity.projectionSemanticsId
expect(ledger.hasUnpublished(id)).toBe(true)
ledger.publishPrefix([id], 2, 2)
expect(ledger.hasUnpublished(id)).toBe(true)
ledger.settlePublishedPrefix('pty-1', 2)
expect(settled).toEqual([])
ledger.publishPrefix([id], 2, 2)
expect(ledger.hasUnpublished(id)).toBe(false)
ledger.settlePublishedPrefix('pty-1', 2)
expect(settled).toEqual([
expect.objectContaining({
spanId: 'span-0',
projectionSemanticsId: id,
sourceStartSu: 0,
sourceEndSu: 4,
displayStart: 0,
displayEnd: 4
})
])
})
it('preserves scanner and display facts after transfer and delivery-token replacement', () => {
const transferred: unknown[] = []
const ledger = new SshPtyLegacyProjectionLedger({
onTransferred: (span) => transferred.push(span)
})
const partial = ledger.commit(
reserve(ledger, {
data: '\x1b[?20',
rawLength: 5,
sequenceEnd: 5,
source: source(0, 5)
})
)
ledger.transfer([partial.identity.projectionSemanticsId], 'renderer-reload')
const continuation = reserve(ledger, {
data: '31h',
rawLength: 3,
sequenceEnd: 8,
source: source(5, 8, 'token-2')
})
expect(continuation.semantics.identity.displayStart).toBe(5)
expect(continuation.semantics.beforeScanner.tail).toBe('\x1b[?20')
expect(continuation.semantics.decision).toBe('subscribed')
expect(transferred).toEqual([
expect.objectContaining({
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 5
})
])
})
it('leaves a projection intact when its terminal transition cannot commit', () => {
const ledger = new SshPtyLegacyProjectionLedger({
onTransferred: () => {
throw new Error('replacement unavailable')
}
})
const projection = ledger.commit(reserve(ledger, { source: source(0, 3) }))
expect(() =>
ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload')
).toThrow('replacement unavailable')
expect(ledger.getDebugSnapshot()).toMatchObject({
transferred: 0,
records: 1
})
})
})
+318
View File
@@ -0,0 +1,318 @@
import { scanMode2031ReplyDecision } from '../../shared/terminal-color-scheme-protocol'
import {
closeProjectionPty,
getOrCreateProjectionCursor,
projectionDebugSnapshot,
projectionError,
reclaimProjectionRecord,
resetProjectionCursorForGap,
requireProjectionRecord,
rollbackCommittedProjectionRecord,
scannerSnapshot,
type DesktopProjectionSpan,
type LegacySshProjectionDebugSnapshot,
type LegacySshProjectionReservation,
type LegacySshProjectionSemantics,
type ProjectionRecord,
type PtyProjectionCursor
} from './ssh-pty-legacy-projection-record'
import {
projectionHasOpen,
resolveProjectionTerminality,
SshPtyProjectionTerminality,
unpublishedProjectionIds
} from './ssh-pty-projection-terminality'
import {
hasUnpublishedLegacyProjection,
publishLegacyProjectionPrefix,
settlePublishedLegacyProjectionPrefix
} from './ssh-pty-legacy-projection-publication'
export type {
DesktopProjectionSpan,
LegacySshProjectionIdentity,
LegacySshProjectionReservation,
LegacySshProjectionSemantics
} from './ssh-pty-legacy-projection-record'
export type SshPtyLegacyProjectionLedgerOptions = {
onSettled?: (span: DesktopProjectionSpan, reason: string) => void
onTransferred?: (span: DesktopProjectionSpan, reason: string) => void
}
export class SshPtyLegacyProjectionLedger {
private nextId = 1
private readonly records = new Map<string, ProjectionRecord>()
private readonly cursorByPty = new Map<string, PtyProjectionCursor>()
private readonly idsByPty = new Map<string, string[]>()
private readonly terminality = new SshPtyProjectionTerminality()
private settledCount = 0
private transferredCount = 0
private rolledBackCount = 0
constructor(private readonly options: SshPtyLegacyProjectionLedgerOptions = {}) {}
reserve(args: {
ptyId: string
providerGeneration: number
ptyIncarnation: string
data: string
sequenceEnd: number
rawLength: number
transformed: boolean
source?: Readonly<{
relayPtyId?: string
spanId: string
clientGeneration: number
ownerGeneration: number
deliveryToken: string
sourceStartSu: number
sourceEndSu: number
}>
}): LegacySshProjectionReservation {
const cursor = getOrCreateProjectionCursor(this.cursorByPty, args, (generation) =>
this.transferGeneration(generation, 'provider-generation-replaced')
)
if (
cursor.providerGeneration !== args.providerGeneration ||
cursor.ptyIncarnation !== args.ptyIncarnation
) {
throw projectionError('ssh_projection_stale_generation')
}
const scan = scanMode2031ReplyDecision(cursor.scanner, args.data)
const projectionSemanticsId = `ssh-projection:${args.providerGeneration}:${this.nextId++}`
const identity = Object.freeze({
projectionSemanticsId,
ptyId: args.ptyId,
providerGeneration: args.providerGeneration,
ptyIncarnation: args.ptyIncarnation,
displayStart: cursor.displayEnd,
displayEnd: cursor.displayEnd + args.data.length,
sequenceEnd: args.sequenceEnd,
rawLength: args.rawLength,
transformed: args.transformed
})
const semantics = Object.freeze({
identity,
...(args.source
? {
desktopSpan: Object.freeze({
...args.source,
id: args.source.relayPtyId ?? args.ptyId,
projectionSemanticsId,
providerGeneration: args.providerGeneration,
ptyIncarnation: args.ptyIncarnation,
displayStart: identity.displayStart,
displayEnd: identity.displayEnd,
splittable: !args.transformed,
transform: Object.freeze({
transformed: args.transformed,
rawLengthSu: args.rawLength,
scalarSafe: !args.transformed
})
})
}
: {}),
beforeScanner: scannerSnapshot(cursor.scanner),
afterScanner: scannerSnapshot(scan.state),
decision: scan.decision
})
this.records.set(projectionSemanticsId, {
semantics,
state: 'reserved',
publishedDisplay: 0,
publishedAccounting: 0,
settledAccounting: 0
})
return Object.freeze({ semantics })
}
commit(reservation: LegacySshProjectionReservation): LegacySshProjectionSemantics {
const record = requireProjectionRecord(
this.records,
reservation.semantics.identity.projectionSemanticsId
)
if (record.state !== 'reserved') {
throw projectionError('ssh_projection_commit_invalid')
}
const { identity, afterScanner } = record.semantics
const cursor = this.cursorByPty.get(identity.ptyId)
if (
!cursor ||
cursor.providerGeneration !== identity.providerGeneration ||
cursor.ptyIncarnation !== identity.ptyIncarnation ||
cursor.displayEnd !== identity.displayStart
) {
this.records.delete(identity.projectionSemanticsId)
throw projectionError('ssh_projection_commit_stale')
}
cursor.displayEnd = identity.displayEnd
cursor.scanner = { ...afterScanner }
record.state = 'committed'
const ids = this.idsByPty.get(identity.ptyId) ?? []
ids.push(identity.projectionSemanticsId)
this.idsByPty.set(identity.ptyId, ids)
return record.semantics
}
rollback(reservation: LegacySshProjectionReservation): boolean {
const id = reservation.semantics.identity.projectionSemanticsId
const record = this.records.get(id)
if (!record || record.state !== 'reserved') {
return false
}
this.records.delete(id)
this.rolledBackCount++
return true
}
rollbackCommitted(reservation: LegacySshProjectionReservation): boolean {
const ptyId = rollbackCommittedProjectionRecord(
this.records,
this.idsByPty,
this.cursorByPty,
reservation
)
if (!ptyId) {
return false
}
this.rolledBackCount++
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
return true
}
publishPrefix(ids: readonly string[], displayChars: number, accountingChars: number): void {
publishLegacyProjectionPrefix(this.records, ids, displayChars, accountingChars)
}
settlePublishedPrefix(ptyId: string, accountingChars: number): number {
const result = settlePublishedLegacyProjectionPrefix(
this.records,
this.idsByPty,
ptyId,
accountingChars,
this.options.onSettled
)
this.settledCount += result.completed
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
return result.settled
}
transfer(ids: readonly string[], reason: string): number {
let transferred = 0
const touchedPtys = new Set<string>()
for (const id of ids.slice()) {
const record = this.records.get(id)
if (!record || record.state === 'reserved') {
continue
}
const ptyId = record.semantics.identity.ptyId
if (record.semantics.desktopSpan) {
this.options.onTransferred?.(record.semantics.desktopSpan, reason)
}
this.transferredCount++
reclaimProjectionRecord(this.records, this.idsByPty, id, ptyId)
touchedPtys.add(ptyId)
transferred++
}
for (const ptyId of touchedPtys) {
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
}
return transferred
}
whenPtyTerminal(
ptyId: string,
providerGeneration: number,
ptyIncarnation: string
): Promise<void> {
return this.terminality.whenTerminal(
ptyId,
providerGeneration,
ptyIncarnation,
projectionHasOpen(this.records, this.idsByPty, ptyId)
)
}
transferGeneration(providerGeneration: number, reason: string): number {
const ids: string[] = []
for (const [id, record] of this.records) {
if (record.semantics.identity.providerGeneration === providerGeneration) {
ids.push(id)
}
}
return this.transfer(ids, reason)
}
transferPty(ptyId: string, reason: string): number {
return this.transfer(this.idsByPty.get(ptyId) ?? [], reason)
}
transferUnpublishedPty(
ptyId: string,
providerGeneration: number,
ptyIncarnation: string,
reason: string
): number {
const ids = unpublishedProjectionIds(
this.records,
this.idsByPty.get(ptyId) ?? [],
providerGeneration,
ptyIncarnation
)
return this.transfer(ids, reason)
}
closePty(
ptyId: string,
providerGeneration: number,
ptyIncarnation: string,
reason: string
): void {
closeProjectionPty(this.cursorByPty, ptyId, providerGeneration, ptyIncarnation, () => {
this.transferPty(ptyId, reason)
this.idsByPty.delete(ptyId)
})
}
closeGeneration(providerGeneration: number, reason: string): void {
for (const [id, record] of this.records) {
if (record.semantics.identity.providerGeneration !== providerGeneration) {
continue
}
if (record.state === 'reserved') {
this.records.delete(id)
this.rolledBackCount++
} else {
this.transfer([id], reason)
}
}
for (const [ptyId, cursor] of this.cursorByPty) {
if (cursor.providerGeneration === providerGeneration) {
this.cursorByPty.delete(ptyId)
this.idsByPty.delete(ptyId)
}
}
this.terminality.closeGeneration(providerGeneration)
}
resetForGap(ptyId: string): void {
resetProjectionCursorForGap(this.cursorByPty, ptyId)
}
get(id: string): LegacySshProjectionSemantics | undefined {
return this.records.get(id)?.semantics
}
hasUnpublished(id: string): boolean {
return hasUnpublishedLegacyProjection(this.records, id)
}
getDebugSnapshot(): LegacySshProjectionDebugSnapshot {
return projectionDebugSnapshot(this.records, this.cursorByPty, {
settled: this.settledCount,
transferred: this.transferredCount,
rolledBack: this.rolledBackCount
})
}
}
@@ -0,0 +1,35 @@
export type SshPtyModelAdmissionKey = Readonly<{
ptyId: string
providerGeneration: number
}>
export type SshPtyModelAdmissionReceipt = Readonly<{
ptyId: string
providerGeneration: number
sequence: number
}>
export type SshPtyModelAdmissionDebugSnapshot = Readonly<{
sourceUnits: number
bytes: number
pressureFrames: number
pressureBytes: number
pausedPtys: number
migratingPtys: number
}>
export type SshPtyModelAdmissionOptions = {
perPtyHighSourceUnits?: number
perPtyHighBytes?: number
perPtyLowSourceUnits?: number
perPtyLowBytes?: number
globalHighSourceUnits?: number
globalHighBytes?: number
globalLowSourceUnits?: number
globalLowBytes?: number
pressureMaxFrames?: number
pressureMaxBytes?: number
pauseProvider?: (key: SshPtyModelAdmissionKey) => boolean
resumeProvider?: (key: SshPtyModelAdmissionKey) => void
closeProvider?: (providerGeneration: number, reason: string) => void
}
@@ -0,0 +1,151 @@
import type {
SshPtyModelAdmissionKey,
SshPtyModelAdmissionReceipt
} from './ssh-pty-model-admission-contract'
import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
export type AdmissionCharge = { sourceUnits: number; bytes: number }
export type AdmissionEntry = {
key: SshPtyModelAdmissionKey
charge: AdmissionCharge
run: () => { sequence: number; completion: Promise<void> }
resolve: (receipt: SshPtyModelAdmissionReceipt) => void
reject: (error: Error) => void
state: 'queued' | 'running' | 'pressure' | 'settled'
}
export type PtyUsage = AdmissionCharge & {
queued: AdmissionEntry[]
running: AdmissionEntry | null
}
export function admissionError(code: string): Error {
return Object.assign(new Error(code), { code })
}
export function retainedBytes(data: string): number {
return Math.max(Buffer.byteLength(data, 'utf8'), 2 * data.length) + 128
}
export function admissionKeyId(key: SshPtyModelAdmissionKey): string {
return `${key.providerGeneration}\0${key.ptyId}`
}
export function canReserveAdmission(args: {
key: SshPtyModelAdmissionKey
charge: AdmissionCharge
limits: SshPtyModelAdmissionLimits
usageByPty: ReadonlyMap<string, PtyUsage>
closingGenerations: ReadonlySet<number>
globalSourceUnits: number
globalBytes: number
}): boolean {
if (args.closingGenerations.has(args.key.providerGeneration)) {
return false
}
const usage = args.usageByPty.get(admissionKeyId(args.key))
return (
(usage?.sourceUnits ?? 0) + args.charge.sourceUnits <= args.limits.perPtyHighSourceUnits &&
(usage?.bytes ?? 0) + args.charge.bytes <= args.limits.perPtyHighBytes &&
args.globalSourceUnits + args.charge.sourceUnits <= args.limits.globalHighSourceUnits &&
args.globalBytes + args.charge.bytes <= args.limits.globalHighBytes
)
}
export function pressureHasAdmissionKey(
entries: readonly AdmissionEntry[],
key: SshPtyModelAdmissionKey
): boolean {
const id = admissionKeyId(key)
return entries.some((entry) => admissionKeyId(entry.key) === id)
}
export function takePressureEntriesForGeneration(
entries: AdmissionEntry[],
providerGeneration: number
): AdmissionEntry[] {
const removed: AdmissionEntry[] = []
for (let index = entries.length - 1; index >= 0; index--) {
if (entries[index]!.key.providerGeneration === providerGeneration) {
removed.unshift(entries.splice(index, 1)[0]!)
}
}
return removed
}
export function cancelAdmissionGeneration(args: {
pressure: AdmissionEntry[]
usageByPty: Map<string, PtyUsage>
idleWaiters: Map<string, Set<() => void>>
providerGeneration: number
error: Error
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
}): number {
let pressureBytes = 0
for (const entry of takePressureEntriesForGeneration(args.pressure, args.providerGeneration)) {
pressureBytes += entry.charge.bytes
entry.state = 'settled'
entry.reject(args.error)
}
for (const [id, usage] of args.usageByPty) {
const canceled = usage.queued.filter(
(entry) => entry.key.providerGeneration === args.providerGeneration
)
usage.queued = usage.queued.filter(
(entry) => entry.key.providerGeneration !== args.providerGeneration
)
if (usage.running?.key.providerGeneration === args.providerGeneration) {
canceled.push(usage.running)
usage.running = null
}
for (const entry of canceled) {
if (entry.state === 'settled') {
continue
}
entry.state = 'settled'
args.release(entry.key, entry.charge)
entry.reject(args.error)
}
if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) {
args.usageByPty.delete(id)
}
resolveAdmissionIdleWaiters(args.usageByPty, args.pressure, args.idleWaiters, id)
}
return pressureBytes
}
export function takePausedGeneration(
paused: Map<string, SshPtyModelAdmissionKey>,
providerGeneration: number
): SshPtyModelAdmissionKey[] {
const removed: SshPtyModelAdmissionKey[] = []
for (const [id, key] of paused) {
if (key.providerGeneration === providerGeneration) {
paused.delete(id)
removed.push(key)
}
}
return removed
}
export function resolveAdmissionIdleWaiters(
usageByPty: ReadonlyMap<string, PtyUsage>,
pressure: readonly AdmissionEntry[],
waitersById: Map<string, Set<() => void>>,
id: string
): void {
const usage = usageByPty.get(id)
const stillPressured = pressure.some((entry) => admissionKeyId(entry.key) === id)
if (usage?.running || (usage?.queued.length ?? 0) > 0 || stillPressured) {
return
}
const waiters = waitersById.get(id)
if (!waiters) {
return
}
waitersById.delete(id)
for (const resolve of waiters) {
resolve()
}
}
@@ -0,0 +1,38 @@
import type { SshPtyModelAdmissionOptions } from './ssh-pty-model-admission-contract'
export type SshPtyModelAdmissionLimits = Readonly<{
perPtyHighSourceUnits: number
perPtyHighBytes: number
perPtyLowSourceUnits: number
perPtyLowBytes: number
globalHighSourceUnits: number
globalHighBytes: number
globalLowSourceUnits: number
globalLowBytes: number
pressureMaxFrames: number
pressureMaxBytes: number
}>
const DEFAULT_PER_PTY_HIGH_SOURCE_UNITS = 256 * 1024
const DEFAULT_PER_PTY_HIGH_BYTES = 2 * 1024 * 1024
const DEFAULT_GLOBAL_HIGH_SOURCE_UNITS = 50 * DEFAULT_PER_PTY_HIGH_SOURCE_UNITS
const DEFAULT_GLOBAL_HIGH_BYTES = 64 * 1024 * 1024
export function resolveSshPtyModelAdmissionLimits(
options: SshPtyModelAdmissionOptions
): SshPtyModelAdmissionLimits {
const perPtyHighSourceUnits = options.perPtyHighSourceUnits ?? DEFAULT_PER_PTY_HIGH_SOURCE_UNITS
const perPtyHighBytes = options.perPtyHighBytes ?? DEFAULT_PER_PTY_HIGH_BYTES
return {
perPtyHighSourceUnits,
perPtyHighBytes,
perPtyLowSourceUnits: options.perPtyLowSourceUnits ?? Math.floor(perPtyHighSourceUnits / 2),
perPtyLowBytes: options.perPtyLowBytes ?? Math.floor(perPtyHighBytes / 2),
globalHighSourceUnits: options.globalHighSourceUnits ?? DEFAULT_GLOBAL_HIGH_SOURCE_UNITS,
globalHighBytes: options.globalHighBytes ?? DEFAULT_GLOBAL_HIGH_BYTES,
globalLowSourceUnits: options.globalLowSourceUnits ?? 8 * 1024 * 1024,
globalLowBytes: options.globalLowBytes ?? 48 * 1024 * 1024,
pressureMaxFrames: options.pressureMaxFrames ?? 64,
pressureMaxBytes: options.pressureMaxBytes ?? 1024 * 1024
}
}
@@ -0,0 +1,91 @@
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
import type { AdmissionCharge, AdmissionEntry, PtyUsage } from './ssh-pty-model-admission-entry'
import { admissionError, admissionKeyId } from './ssh-pty-model-admission-entry'
import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
export function beginSshPtyModelAdmissionMigration(args: {
key: SshPtyModelAdmissionKey
migratingPtys: Set<string>
pressure: SshPtyModelAdmissionPressure
usageByPty: Map<string, PtyUsage>
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
cleanup: (id: string, usage: PtyUsage) => void
}): void {
const id = admissionKeyId(args.key)
if (args.migratingPtys.has(id)) {
return
}
args.migratingPtys.add(id)
const error = admissionError('ssh_model_migration_queued_canceled')
args.pressure.cancelQueuedPty(args.key, error)
const usage = args.usageByPty.get(id)
if (!usage) {
return
}
const queued = usage.queued
usage.queued = []
for (const entry of queued) {
cancelQueuedEntry(entry, error, args.release)
}
args.cleanup(id, usage)
}
export function closeSshPtyModelAdmissionMigrations(
migratingPtys: Set<string>,
providerGeneration: number
): void {
const prefix = `${providerGeneration}\0`
for (const id of migratingPtys) {
if (id.startsWith(prefix)) {
migratingPtys.delete(id)
}
}
}
export function settleSshPtyModelAdmissionFailure(args: {
id: string
usage: PtyUsage
entry: AdmissionEntry
error: Error
migratingPtys: ReadonlySet<string>
closingGenerations: Set<number>
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
closeGeneration: (providerGeneration: number) => void
cleanup: (id: string, usage: PtyUsage) => void
}): void {
if (args.entry.state !== 'running' || args.usage.running !== args.entry) {
return
}
const migrationOwnsFailure = args.migratingPtys.has(args.id)
if (!migrationOwnsFailure) {
args.closingGenerations.add(args.entry.key.providerGeneration)
}
args.usage.running = null
args.entry.state = 'settled'
args.release(args.entry.key, args.entry.charge)
args.entry.reject(migrationOwnsFailure ? migrationCompletionError(args.error) : args.error)
if (!migrationOwnsFailure) {
args.closeGeneration(args.entry.key.providerGeneration)
}
args.cleanup(args.id, args.usage)
}
function migrationCompletionError(cause: Error): Error {
return Object.assign(new Error(cause.message), {
code: 'ssh_model_migration_completion_failed',
cause
})
}
function cancelQueuedEntry(
entry: AdmissionEntry,
error: Error,
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
): void {
if (entry.state === 'settled') {
return
}
entry.state = 'settled'
release(entry.key, entry.charge)
entry.reject(error)
}
@@ -0,0 +1,153 @@
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
import {
admissionKeyId,
pressureHasAdmissionKey,
takePausedGeneration,
type AdmissionEntry,
type PtyUsage
} from './ssh-pty-model-admission-entry'
import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
type PressureOptions = {
limits: SshPtyModelAdmissionLimits
pauseProvider: (key: SshPtyModelAdmissionKey) => boolean
resumeProvider: (key: SshPtyModelAdmissionKey) => void
}
type PromotionOptions = {
usageByPty: ReadonlyMap<string, PtyUsage>
disposed: boolean
canReserve: (entry: AdmissionEntry) => boolean
reserve: (entry: AdmissionEntry) => void
isBelowGlobalLowWatermark: () => boolean
}
export class SshPtyModelAdmissionPressure {
private readonly entries: AdmissionEntry[] = []
private readonly pausedKeys = new Map<string, SshPtyModelAdmissionKey>()
private retainedBytes = 0
constructor(private readonly options: PressureOptions) {}
get values(): readonly AdmissionEntry[] {
return this.entries
}
get frameCount(): number {
return this.entries.length
}
get bytes(): number {
return this.retainedBytes
}
get pausedPtyCount(): number {
return this.pausedKeys.size
}
get pausedProviderGenerations(): ReadonlySet<number> {
return new Set(Array.from(this.pausedKeys.values(), (key) => key.providerGeneration))
}
has(key: SshPtyModelAdmissionKey): boolean {
return pressureHasAdmissionKey(this.entries, key)
}
admit(entry: AdmissionEntry): boolean {
const id = admissionKeyId(entry.key)
const paused = this.pausedKeys.has(id) || this.options.pauseProvider(entry.key)
if (paused) {
this.pausedKeys.set(id, entry.key)
}
if (
!paused ||
this.entries.length >= this.options.limits.pressureMaxFrames ||
this.retainedBytes + entry.charge.bytes > this.options.limits.pressureMaxBytes
) {
return false
}
entry.state = 'pressure'
this.entries.push(entry)
this.retainedBytes += entry.charge.bytes
return true
}
cancelGeneration(
providerGeneration: number,
cancelPressureAndReserved: (entries: AdmissionEntry[]) => number
): void {
this.retainedBytes -= cancelPressureAndReserved(this.entries)
for (const key of takePausedGeneration(this.pausedKeys, providerGeneration)) {
this.resume(key)
}
}
cancelPty(key: SshPtyModelAdmissionKey, error: Error, cancelReserved: () => void): void {
this.cancelQueuedPty(key, error)
cancelReserved()
}
cancelQueuedPty(key: SshPtyModelAdmissionKey, error: Error): void {
const id = admissionKeyId(key)
const canceled: AdmissionEntry[] = []
for (let index = this.entries.length - 1; index >= 0; index--) {
if (admissionKeyId(this.entries[index]!.key) === id) {
canceled.push(this.entries.splice(index, 1)[0]!)
}
}
this.rejectPressureEntries(canceled, error)
const paused = this.pausedKeys.get(id)
if (paused) {
this.pausedKeys.delete(id)
this.resume(paused)
}
}
promoteAndResume(options: PromotionOptions): void {
if (options.disposed) {
return
}
for (let index = 0; index < this.entries.length; ) {
const entry = this.entries[index]!
const hasEarlierEntryForPty = this.entries
.slice(0, index)
.some((earlier) => admissionKeyId(earlier.key) === admissionKeyId(entry.key))
if (hasEarlierEntryForPty || !options.canReserve(entry)) {
index++
continue
}
this.entries.splice(index, 1)
this.retainedBytes -= entry.charge.bytes
options.reserve(entry)
}
if (!options.isBelowGlobalLowWatermark()) {
return
}
for (const [id, key] of this.pausedKeys) {
const usage = options.usageByPty.get(id)
if (
this.has(key) ||
(usage?.sourceUnits ?? 0) > this.options.limits.perPtyLowSourceUnits ||
(usage?.bytes ?? 0) > this.options.limits.perPtyLowBytes
) {
continue
}
this.pausedKeys.delete(id)
this.resume(key)
}
}
private rejectPressureEntries(entries: readonly AdmissionEntry[], error: Error): void {
for (const entry of entries) {
this.retainedBytes -= entry.charge.bytes
entry.state = 'settled'
entry.reject(error)
}
}
private resume(key: SshPtyModelAdmissionKey): void {
try {
this.options.resumeProvider(key)
} catch {}
}
}
@@ -0,0 +1,18 @@
import type { SshPtyModelAdmissionDebugSnapshot } from './ssh-pty-model-admission-contract'
import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
export function sshPtyModelAdmissionSnapshot(
sourceUnits: number,
bytes: number,
pressure: SshPtyModelAdmissionPressure,
migratingPtys: ReadonlySet<string>
): SshPtyModelAdmissionDebugSnapshot {
return {
sourceUnits,
bytes,
pressureFrames: pressure.frameCount,
pressureBytes: pressure.bytes,
pausedPtys: pressure.pausedPtyCount,
migratingPtys: migratingPtys.size
}
}
@@ -0,0 +1,150 @@
import { describe, expect, it, vi } from 'vitest'
import { SshPtyModelAdmission } from './ssh-pty-model-admission'
function deferred() {
let resolve!: () => void
let reject!: (error: Error) => void
const promise = new Promise<void>((promiseResolve, promiseReject) => {
resolve = promiseResolve
reject = promiseReject
})
return { promise, resolve, reject }
}
function accept(admission: SshPtyModelAdmission, completion: Promise<void>) {
return admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'data', 4, () => ({
sequence: 4,
completion
}))
}
describe('SshPtyModelAdmission', () => {
it('freezes migration while retaining the running raw completion', async () => {
const runningCompletion = deferred()
const admission = new SshPtyModelAdmission()
const running = accept(admission, runningCompletion.promise)
const queued = accept(admission, Promise.resolve())
admission.beginMigration({ ptyId: 'pty-1', providerGeneration: 7 })
await expect(queued).rejects.toThrow('ssh_model_migration_queued_canceled')
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 4 })
await expect(
admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'late', 4, () => ({
sequence: 8,
completion: Promise.resolve()
}))
).rejects.toThrow('ssh_model_admission_migrating')
runningCompletion.resolve()
await expect(running).resolves.toMatchObject({ sequence: 4 })
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
})
it('cancels a never-settling running entry when its generation closes', async () => {
const admission = new SshPtyModelAdmission()
const receipt = accept(admission, new Promise<void>(() => {}))
const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 })
admission.closeGeneration(7, 'provider-closed')
await expect(receipt).rejects.toThrow('provider-closed')
await expect(idle).resolves.toBeUndefined()
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
})
it('cancels a never-settling running entry on disposal', async () => {
const admission = new SshPtyModelAdmission()
const receipt = accept(admission, new Promise<void>(() => {}))
const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 })
admission.dispose()
await expect(receipt).rejects.toThrow('ssh_model_admission_disposed')
await expect(idle).resolves.toBeUndefined()
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
})
it('keeps non-migrating callback failure generation-fatal across sibling PTYs', async () => {
const admission = new SshPtyModelAdmission()
const failedCompletion = deferred()
const siblingCompletion = deferred()
const failed = accept(admission, failedCompletion.promise)
const sibling = admission.accept({ ptyId: 'pty-2', providerGeneration: 7 }, 'data', 4, () => ({
sequence: 4,
completion: siblingCompletion.promise
}))
failedCompletion.reject(new Error('emulator failed'))
await expect(failed).rejects.toThrow('emulator failed')
await expect(sibling).rejects.toThrow('ssh_model_admission_completion_failed')
await expect(
admission.accept({ ptyId: 'pty-3', providerGeneration: 7 }, 'data', 4, () => ({
sequence: 4,
completion: Promise.resolve()
}))
).rejects.toThrow('ssh_model_admission_generation_closed')
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
siblingCompletion.resolve()
await Promise.resolve()
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
})
it('resumes every paused provider generation exactly once on disposal', async () => {
const resumeProvider = vi.fn()
const admission = new SshPtyModelAdmission({
perPtyHighSourceUnits: 4,
perPtyHighBytes: 1024,
globalHighSourceUnits: 4,
globalHighBytes: 1024,
pressureMaxFrames: 1,
pressureMaxBytes: 1024,
pauseProvider: () => true,
resumeProvider
})
const running = accept(admission, new Promise<void>(() => {}))
const pressured = accept(admission, Promise.resolve())
const rejected = admission.accept({ ptyId: 'pty-2', providerGeneration: 8 }, 'data', 4, () => ({
sequence: 4,
completion: Promise.resolve()
}))
await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted')
admission.dispose()
admission.dispose()
await expect(running).rejects.toThrow('ssh_model_admission_disposed')
await expect(pressured).rejects.toThrow('ssh_model_admission_disposed')
expect(resumeProvider.mock.calls).toEqual([
[{ ptyId: 'pty-1', providerGeneration: 7 }],
[{ ptyId: 'pty-2', providerGeneration: 8 }]
])
})
it.each(['resolve', 'reject'] as const)(
'ignores a late completion %s after cancellation',
async (settle) => {
const completion = deferred()
const admission = new SshPtyModelAdmission()
const onResolve = vi.fn()
const onReject = vi.fn()
const receipt = accept(admission, completion.promise)
const observed = receipt.then(onResolve, onReject)
admission.closeGeneration(7, 'provider-closed')
await observed
if (settle === 'resolve') {
completion.resolve()
} else {
completion.reject(new Error('late emulator failure'))
}
await Promise.resolve()
expect(onResolve).not.toHaveBeenCalled()
expect(onReject).toHaveBeenCalledTimes(1)
expect(onReject.mock.calls[0]?.[0]).toMatchObject({ message: 'provider-closed' })
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
}
)
})
+315
View File
@@ -0,0 +1,315 @@
import type {
SshPtyModelAdmissionKey,
SshPtyModelAdmissionOptions,
SshPtyModelAdmissionReceipt
} from './ssh-pty-model-admission-contract'
import {
admissionError,
admissionKeyId,
canReserveAdmission,
cancelAdmissionGeneration,
retainedBytes,
resolveAdmissionIdleWaiters,
type AdmissionCharge,
type AdmissionEntry,
type PtyUsage
} from './ssh-pty-model-admission-entry'
import { resolveSshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
import * as modelAdmissionMigration from './ssh-pty-model-admission-migration'
import { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
import { sshPtyModelAdmissionSnapshot } from './ssh-pty-model-admission-snapshot'
export type * from './ssh-pty-model-admission-contract'
export class SshPtyModelAdmission {
private readonly limits: ReturnType<typeof resolveSshPtyModelAdmissionLimits>
private readonly closeProvider: (providerGeneration: number, reason: string) => void
private readonly usageByPty = new Map<string, PtyUsage>()
private readonly pressure: SshPtyModelAdmissionPressure
private readonly idleWaiters = new Map<string, Set<() => void>>()
private readonly closingGenerations = new Set<number>()
private readonly migratingPtys = new Set<string>()
private globalSourceUnits = 0
private globalBytes = 0
private disposed = false
constructor(options: SshPtyModelAdmissionOptions = {}) {
this.limits = resolveSshPtyModelAdmissionLimits(options)
this.pressure = new SshPtyModelAdmissionPressure({
limits: this.limits,
pauseProvider: options.pauseProvider ?? (() => false),
resumeProvider: options.resumeProvider ?? (() => {})
})
this.closeProvider = options.closeProvider ?? (() => {})
}
accept(
key: SshPtyModelAdmissionKey,
data: string,
sourceUnits: number,
run: () => { sequence: number; completion: Promise<void> }
): Promise<SshPtyModelAdmissionReceipt> {
if (this.disposed) {
return Promise.reject(admissionError('ssh_model_admission_disposed'))
}
if (this.closingGenerations.has(key.providerGeneration)) {
return Promise.reject(admissionError('ssh_model_admission_generation_closed'))
}
if (this.migratingPtys.has(admissionKeyId(key))) {
return Promise.reject(admissionError('ssh_model_admission_migrating'))
}
const charge = { sourceUnits, bytes: retainedBytes(data) }
return new Promise((resolve, reject) => {
const entry: AdmissionEntry = {
key: { ...key },
charge,
run,
resolve,
reject,
state: 'queued'
}
if (this.canReserve(key, charge) && !this.pressure.has(key)) {
this.reserveAndQueue(entry)
return
}
if (!this.pressure.admit(entry)) {
entry.reject(admissionError('ssh_model_admission_pressure_exhausted'))
this.closeProvider(entry.key.providerGeneration, 'model-admission-pressure')
}
})
}
closeGeneration(providerGeneration: number, reason = 'provider-generation-closed'): void {
this.closingGenerations.add(providerGeneration)
const error = admissionError(reason)
this.pressure.cancelGeneration(providerGeneration, (pressure) =>
cancelAdmissionGeneration({
pressure,
usageByPty: this.usageByPty,
idleWaiters: this.idleWaiters,
providerGeneration,
error,
release: (key, charge) => this.release(key, charge)
})
)
modelAdmissionMigration.closeSshPtyModelAdmissionMigrations(
this.migratingPtys,
providerGeneration
)
const generationPrefix = `${providerGeneration}\0`
for (const id of this.idleWaiters.keys()) {
if (id.startsWith(generationPrefix)) {
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
}
}
}
beginMigration(key: SshPtyModelAdmissionKey): void {
modelAdmissionMigration.beginSshPtyModelAdmissionMigration({
key,
migratingPtys: this.migratingPtys,
pressure: this.pressure,
usageByPty: this.usageByPty,
release: (entryKey, charge) => this.release(entryKey, charge),
cleanup: (id, usage) => this.cleanupUsage(id, usage)
})
const id = admissionKeyId(key)
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
}
cancelPty(key: SshPtyModelAdmissionKey, reason: string): void {
const id = admissionKeyId(key)
const error = admissionError(reason)
this.pressure.cancelPty(key, error, () => {
const usage = this.usageByPty.get(id)
if (usage) {
const canceled = [...usage.queued, ...(usage.running ? [usage.running] : [])]
usage.queued = []
usage.running = null
for (const entry of canceled) {
if (entry.state === 'settled') {
continue
}
entry.state = 'settled'
this.release(entry.key, entry.charge)
entry.reject(error)
}
this.usageByPty.delete(id)
}
})
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
}
whenIdle(key: SshPtyModelAdmissionKey): Promise<void> {
const id = admissionKeyId(key)
const usage = this.usageByPty.get(id)
const hasPressure = this.pressure.has(key)
if ((!usage || (!usage.running && usage.queued.length === 0)) && !hasPressure) {
return Promise.resolve()
}
return new Promise((resolve) => {
const waiters = this.idleWaiters.get(id) ?? new Set<() => void>()
waiters.add(resolve)
this.idleWaiters.set(id, waiters)
})
}
dispose(): void {
if (this.disposed) {
return
}
this.disposed = true
const generations = new Set<number>()
for (const usage of this.usageByPty.values()) {
if (usage.running) {
generations.add(usage.running.key.providerGeneration)
}
for (const entry of usage.queued) {
generations.add(entry.key.providerGeneration)
}
}
for (const entry of this.pressure.values) {
generations.add(entry.key.providerGeneration)
}
for (const generation of this.pressure.pausedProviderGenerations) {
generations.add(generation)
}
for (const generation of generations) {
this.closeGeneration(generation, 'ssh_model_admission_disposed')
}
}
getDebugSnapshot() {
return sshPtyModelAdmissionSnapshot(
this.globalSourceUnits,
this.globalBytes,
this.pressure,
this.migratingPtys
)
}
private canReserve(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): boolean {
if (this.migratingPtys.has(admissionKeyId(key))) {
return false
}
return canReserveAdmission({
key,
charge,
limits: this.limits,
usageByPty: this.usageByPty,
closingGenerations: this.closingGenerations,
globalSourceUnits: this.globalSourceUnits,
globalBytes: this.globalBytes
})
}
private reserveAndQueue(entry: AdmissionEntry): void {
const id = admissionKeyId(entry.key)
let usage = this.usageByPty.get(id)
if (!usage) {
usage = { sourceUnits: 0, bytes: 0, queued: [], running: null }
this.usageByPty.set(id, usage)
}
usage.sourceUnits += entry.charge.sourceUnits
usage.bytes += entry.charge.bytes
this.globalSourceUnits += entry.charge.sourceUnits
this.globalBytes += entry.charge.bytes
entry.state = 'queued'
usage.queued.push(entry)
this.startNext(id, usage)
}
private startNext(id: string, usage: PtyUsage): void {
if (usage.running) {
return
}
const entry = usage.queued.shift()
if (!entry) {
this.promotePressureAndResumeProvider()
return
}
usage.running = entry
entry.state = 'running'
let execution: { sequence: number; completion: Promise<void> }
try {
execution = entry.run()
} catch (error) {
if (this.finishEntry(id, usage, entry)) {
entry.reject(error instanceof Error ? error : new Error(String(error)))
}
return
}
void execution.completion.then(
() => {
if (!this.finishEntry(id, usage, entry)) {
return
}
entry.resolve({
ptyId: entry.key.ptyId,
providerGeneration: entry.key.providerGeneration,
sequence: execution.sequence
})
},
(error) => {
this.failEntry(id, usage, entry, error instanceof Error ? error : new Error(String(error)))
}
)
}
private failEntry(id: string, usage: PtyUsage, entry: AdmissionEntry, error: Error): void {
modelAdmissionMigration.settleSshPtyModelAdmissionFailure({
id,
usage,
entry,
error,
migratingPtys: this.migratingPtys,
closingGenerations: this.closingGenerations,
release: (key, charge) => this.release(key, charge),
closeGeneration: (providerGeneration) =>
this.closeGeneration(providerGeneration, 'ssh_model_admission_completion_failed'),
cleanup: (entryId, entryUsage) => this.cleanupUsage(entryId, entryUsage)
})
}
private finishEntry(id: string, usage: PtyUsage, entry: AdmissionEntry): boolean {
if (entry.state !== 'running' || usage.running !== entry) {
return false
}
usage.running = null
entry.state = 'settled'
this.release(entry.key, entry.charge)
this.cleanupUsage(id, usage)
return true
}
private cleanupUsage(id: string, usage: PtyUsage): void {
this.startNext(id, usage)
if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) {
this.usageByPty.delete(id)
}
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
}
private release(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): void {
const usage = this.usageByPty.get(admissionKeyId(key))
if (usage) {
usage.sourceUnits = Math.max(0, usage.sourceUnits - charge.sourceUnits)
usage.bytes = Math.max(0, usage.bytes - charge.bytes)
}
this.globalSourceUnits = Math.max(0, this.globalSourceUnits - charge.sourceUnits)
this.globalBytes = Math.max(0, this.globalBytes - charge.bytes)
this.promotePressureAndResumeProvider()
}
private promotePressureAndResumeProvider(): void {
this.pressure.promoteAndResume({
usageByPty: this.usageByPty,
disposed: this.disposed,
canReserve: (entry) => this.canReserve(entry.key, entry.charge),
reserve: (entry) => this.reserveAndQueue(entry),
isBelowGlobalLowWatermark: () =>
this.globalSourceUnits <= this.limits.globalLowSourceUnits &&
this.globalBytes <= this.limits.globalLowBytes
})
}
}
@@ -0,0 +1,155 @@
import { describe, expect, it, vi } from 'vitest'
import type { RemoteTerminalSourceRangeStreamIdentity } from '../runtime/remote-terminal-source-range-consumer'
import {
SshPtyOutputIntake,
type SshPtyOutputIntakeDependencies,
type SshPtyOutputReceipt
} from './ssh-pty-output-intake'
import type { SshPtySourceCancellationProof } from './ssh-pty-output-intake-contract'
type ExitDeadlineHarness = Readonly<{
intake: SshPtyOutputIntake
dependencies: SshPtyOutputIntakeDependencies
cancelSourceDelivery: ReturnType<typeof vi.fn>
releaseExit: ReturnType<typeof vi.fn>
exits: string[]
}>
const stream: RemoteTerminalSourceRangeStreamIdentity = {
ptyId: 'pty-1',
consumerId: 'remote-1',
streamGeneration: 'stream-1'
}
function createHarness(
cancelSourceDelivery: NonNullable<SshPtyOutputIntakeDependencies['cancelSourceDelivery']>
): ExitDeadlineHarness {
const exits: string[] = []
const releaseExit = vi.fn()
const cancellation = vi.fn(cancelSourceDelivery)
const dependencies: SshPtyOutputIntakeDependencies = {
getModelSequence: () => 0,
acceptModel: (event) => ({ sequence: event.rawLength, completion: Promise.resolve() }),
project: vi.fn(),
prepareExit: vi.fn(() => releaseExit),
finalizeExit: () => exits.push('exit'),
pauseProvider: vi.fn(() => true),
resumeProvider: vi.fn(),
closeProvider: vi.fn(),
cancelSourceDelivery: cancellation
}
return {
intake: new SshPtyOutputIntake(dependencies, {
exitBarrierMs: 10,
exitCancellationProofMs: 100
}),
dependencies,
cancelSourceDelivery: cancellation,
releaseExit,
exits
}
}
async function publishSource(harness: ExitDeadlineHarness): Promise<SshPtyOutputReceipt> {
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
expect(remote.attach(stream)).toBe(true)
const receipt = await harness.intake.acceptData({
id: 'pty-1',
data: 'aaaa',
providerGeneration: 1,
ptyIncarnation: 'incarnation-1',
rawLength: 4,
transformed: false,
source: {
spanId: 'span-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
harness.intake.publishProjectionPrefix([receipt.projection.identity.projectionSemanticsId], 4, 4)
return receipt
}
function acceptExit(harness: ExitDeadlineHarness) {
return harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
}
describe('SshPtyOutputExitDeadline', () => {
it('transfers published projections before cancellation proof reclaims their spans', async () => {
vi.useFakeTimers()
try {
const harness = createHarness(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
const receipt = await publishSource(harness)
const exitResult = acceptExit(harness).then(
() => ({ ok: true as const }),
(error: Error) => ({ ok: false as const, error })
)
await vi.advanceTimersByTimeAsync(10)
expect(await exitResult).toEqual({ ok: true })
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
expect(harness.dependencies.prepareExit).toHaveBeenCalledOnce()
expect(harness.releaseExit).toHaveBeenCalledOnce()
expect(harness.exits).toEqual(['exit'])
expect(harness.intake.getDebugSnapshot()).toMatchObject({
projection: { records: 0 },
source: { openedTokens: 0, ptyIdentities: 0 },
exitBarriers: 0
})
await vi.advanceTimersByTimeAsync(0)
expect(harness.exits).toEqual(['exit'])
expect(() =>
harness.intake
.getRemoteSourceRangeConsumerHooks()
.settle(stream, [receipt.projection.desktopSpan!])
).not.toThrow()
} finally {
vi.useRealTimers()
}
})
it('generation close fences a pending cancellation proof from final exit', async () => {
vi.useFakeTimers()
try {
let resolveCancellation!: (proof: SshPtySourceCancellationProof) => void
const cancellation = new Promise<SshPtySourceCancellationProof>((resolve) => {
resolveCancellation = resolve
})
const harness = createHarness(() => cancellation)
await publishSource(harness)
const exitResult = acceptExit(harness).then(
() => ({ ok: true as const }),
(error: Error) => ({ ok: false as const, error })
)
await vi.advanceTimersByTimeAsync(10)
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
harness.intake.closeGeneration(1, 'provider-closed')
const closed = await exitResult
expect(closed.ok).toBe(false)
expect(closed.ok ? '' : closed.error.message).toContain('provider-closed')
resolveCancellation({ sentEndSu: 4, creditedEndSu: 0 })
await vi.advanceTimersByTimeAsync(0)
expect(harness.exits).toEqual([])
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
expect(harness.intake.getDebugSnapshot()).toMatchObject({
projection: { records: 0 },
source: { openedTokens: 0, ptyIdentities: 0 },
exitBarriers: 0
})
} finally {
vi.useRealTimers()
}
})
})
@@ -0,0 +1,239 @@
import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
import type {
SshPtyOutputExitEvent,
SshPtyOutputIntakeDependencies
} from './ssh-pty-output-intake-contract'
import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation'
import type {
SshPtyOutputSourceObligations,
SshPtySourceCancellationProofCommit
} from './ssh-pty-output-source-obligations'
type SshPtyOutputExitDeadlineDependencies = Readonly<{
admission: SshPtyModelAdmission
projections: SshPtyLegacyProjectionLedger
sourceObligations: SshPtyOutputSourceObligations
intake: SshPtyOutputIntakeDependencies
barrierMs?: number
cancellationProofMs?: number
}>
export class SshPtyOutputExitDeadline {
private readonly barriersByGeneration = new Map<number, Set<SshPtyExitBarrier>>()
private readonly preparedExits = new Set<string>()
private readonly preparedExitReleases = new Map<string, () => void>()
private readonly barrierMs: number
private readonly cancellationProofMs: number
constructor(private readonly dependencies: SshPtyOutputExitDeadlineDependencies) {
this.barrierMs = dependencies.barrierMs ?? 30_000
this.cancellationProofMs = dependencies.cancellationProofMs ?? 10_000
}
wait(
event: SshPtyOutputExitEvent,
start: (validateNormalExit: () => void) => Promise<void>
): Promise<void> {
return new Promise((resolve, reject) => {
let timeoutStarted = false
let settled = false
const validateNormalExit = (): void => {
if (timeoutStarted) {
throw outputIntakeError('ssh_exit_delivery_canceled')
}
}
const settle = (result: { ok: true } | { ok: false; error: Error }): void => {
if (settled) {
return
}
settled = true
clearTimeout(barrier.timer)
this.releasePreparedExit(event)
this.remove(event.providerGeneration, barrier)
if (result.ok) {
resolve()
} else {
reject(result.error)
}
}
const barrier: SshPtyExitBarrier = {
timer: setTimeout(() => {
timeoutStarted = true
void this.cancelTimedOutExit(event, barrier).then(
() => settle({ ok: true }),
(error) => {
this.dependencies.intake.closeProvider?.(
event.providerGeneration,
'ssh-exit-cancellation-proof-failed'
)
settle({
ok: false,
error: error instanceof Error ? error : outputIntakeError(String(error))
})
}
)
}, this.barrierMs),
reject: (error) => settle({ ok: false, error })
}
barrier.timer.unref?.()
let barriers = this.barriersByGeneration.get(event.providerGeneration)
if (!barriers) {
barriers = new Set()
this.barriersByGeneration.set(event.providerGeneration, barriers)
}
barriers.add(barrier)
const promise = start(validateNormalExit)
void promise.then(
() => {
if (!timeoutStarted) {
settle({ ok: true })
}
},
(error) => {
if (!timeoutStarted) {
settle({ ok: false, error })
}
}
)
})
}
closeGeneration(providerGeneration: number, error: Error): void {
const barriers = this.barriersByGeneration.get(providerGeneration)
if (barriers) {
for (const barrier of barriers) {
clearTimeout(barrier.timer)
barrier.reject(error)
}
this.barriersByGeneration.delete(providerGeneration)
}
const prefix = `${providerGeneration}\0`
for (const key of this.preparedExits) {
if (key.startsWith(prefix)) {
this.releasePreparedExitKey(key)
this.preparedExits.delete(key)
}
}
}
prepareExitOnce(event: SshPtyOutputExitEvent): void {
const key = this.exitKey(event)
if (this.preparedExits.has(key)) {
return
}
const release = this.dependencies.intake.prepareExit(event)
this.preparedExits.add(key)
if (release) {
this.preparedExitReleases.set(key, release)
}
}
get activeBarriers(): number {
return Array.from(this.barriersByGeneration.values()).reduce(
(total, barriers) => total + barriers.size,
0
)
}
private async cancelTimedOutExit(
event: SshPtyOutputExitEvent,
barrier: SshPtyExitBarrier
): Promise<void> {
const cancel = this.dependencies.intake.cancelSourceDelivery
if (!cancel) {
throw outputIntakeError('ssh_source_cancellation_publisher_unavailable')
}
this.dependencies.admission.cancelPty(
{ ptyId: event.id, providerGeneration: event.providerGeneration },
'ssh_exit_delivery_canceled'
)
this.dependencies.sourceObligations.sealPty(event)
const cancellation = this.dependencies.sourceObligations.requestPtyCancellationProof(
event,
(request) => cancel(event.providerGeneration, request)
)
let commit: SshPtySourceCancellationProofCommit | null
try {
commit = await this.withCancellationProofDeadline(cancellation)
} catch (error) {
if (!this.isActive(event.providerGeneration, barrier)) {
return
}
throw error
}
if (!this.isActive(event.providerGeneration, barrier)) {
return
}
if (!commit) {
throw outputIntakeError('ssh_source_cancellation_identity_unavailable')
}
this.dependencies.projections.transferPty(event.id, 'ssh-exit-delivery-canceled')
this.dependencies.sourceObligations.commitPtyCancellationProof(commit)
this.prepareExitOnce(event)
if (!this.isActive(event.providerGeneration, barrier)) {
return
}
this.dependencies.intake.finalizeExit(event)
if (!this.isActive(event.providerGeneration, barrier)) {
return
}
this.dependencies.projections.closePty(
event.id,
event.providerGeneration,
event.ptyIncarnation,
'ssh-exit-delivery-canceled'
)
}
private isActive(providerGeneration: number, barrier: SshPtyExitBarrier): boolean {
return this.barriersByGeneration.get(providerGeneration)?.has(barrier) ?? false
}
private exitKey(event: SshPtyOutputExitEvent): string {
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
}
private releasePreparedExit(event: SshPtyOutputExitEvent): void {
const key = this.exitKey(event)
this.preparedExits.delete(key)
this.releasePreparedExitKey(key)
}
private releasePreparedExitKey(key: string): void {
const release = this.preparedExitReleases.get(key)
if (!release) {
return
}
this.preparedExitReleases.delete(key)
release()
}
private withCancellationProofDeadline<T>(promise: Promise<T>): Promise<T> {
return new Promise((resolve, reject) => {
const timer = setTimeout(
() => reject(outputIntakeError('ssh_source_cancellation_proof_timeout')),
this.cancellationProofMs
)
timer.unref?.()
void promise.then(
(value) => {
clearTimeout(timer)
resolve(value)
},
(error) => {
clearTimeout(timer)
reject(error)
}
)
})
}
private remove(providerGeneration: number, barrier: SshPtyExitBarrier): void {
const barriers = this.barriersByGeneration.get(providerGeneration)
barriers?.delete(barrier)
if (barriers?.size === 0) {
this.barriersByGeneration.delete(providerGeneration)
}
}
}
+75
View File
@@ -0,0 +1,75 @@
import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
import type {
SshPtyOutputExitEvent,
SshPtyOutputIntakeDependencies
} from './ssh-pty-output-intake-contract'
export async function settleSshPtyOutputExit(args: {
event: SshPtyOutputExitEvent
admission: SshPtyModelAdmission
projections: SshPtyLegacyProjectionLedger
dependencies: SshPtyOutputIntakeDependencies
validateGeneration: () => void
prepareExit?: () => void
afterAdmissionIdle?: () => void
waitForSourceTerminal?: () => Promise<void>
beforeFinalize?: () => void
}): Promise<void> {
const {
event,
admission,
projections,
dependencies,
validateGeneration,
prepareExit,
afterAdmissionIdle,
waitForSourceTerminal,
beforeFinalize
} = args
await admission.whenIdle({
ptyId: event.id,
providerGeneration: event.providerGeneration
})
validateGeneration()
afterAdmissionIdle?.()
try {
if (prepareExit) {
prepareExit()
} else {
dependencies.prepareExit(event)
}
} catch (error) {
projections.closePty(
event.id,
event.providerGeneration,
event.ptyIncarnation,
'pty-exit-finalize-failed'
)
dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed')
throw error
}
projections.transferUnpublishedPty(
event.id,
event.providerGeneration,
event.ptyIncarnation,
'pty-exit-unpublished'
)
await projections.whenPtyTerminal(event.id, event.providerGeneration, event.ptyIncarnation)
await waitForSourceTerminal?.()
validateGeneration()
try {
beforeFinalize?.()
dependencies.finalizeExit(event)
projections.closePty(event.id, event.providerGeneration, event.ptyIncarnation, 'pty-exit')
} catch (error) {
projections.closePty(
event.id,
event.providerGeneration,
event.ptyIncarnation,
'pty-exit-finalize-failed'
)
dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed')
throw error
}
}
@@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest'
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
const event = (providerGeneration: number) => ({
id: 'pty-1',
providerGeneration,
ptyIncarnation: 'incarnation-1'
})
describe('SshPtyOutputGenerationGuard', () => {
it('compacts sequential closures without weakening stale rejection', () => {
const guard = new SshPtyOutputGenerationGuard(() => false)
for (let generation = 1; generation <= 2_048; generation++) {
guard.closeGeneration(generation)
}
expect(guard.getDebugSnapshot()).toEqual({
closedRanges: 1,
activeGaps: 0,
activePtys: 0,
sealedPtys: 0
})
expect(() => guard.validate(event(1_024))).toThrow('ssh_output_stale_generation')
expect(() => guard.validate(event(2_049))).not.toThrow()
})
it('counts and merges out-of-order live generation gaps exactly', () => {
const guard = new SshPtyOutputGenerationGuard(() => false)
guard.closeGeneration(4)
guard.closeGeneration(2)
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 2, activeGaps: 2 })
guard.closeGeneration(3)
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 1, activeGaps: 1 })
expect(() => guard.validate(event(1))).not.toThrow()
expect(() => guard.validate(event(2))).toThrow('ssh_output_stale_generation')
guard.closeGeneration(1)
expect(guard.getDebugSnapshot()).toEqual({
closedRanges: 1,
activeGaps: 0,
activePtys: 0,
sealedPtys: 0
})
})
})
@@ -0,0 +1,107 @@
import type { SshPtyOutputDataEvent, SshPtyOutputExitEvent } from './ssh-pty-output-intake-contract'
import {
outputIntakeError,
sshPtyGenerationKey,
validOutputLength
} from './ssh-pty-output-intake-validation'
import { SshPtyClosedGenerationRanges } from './ssh-pty-closed-generation-ranges'
export class SshPtyOutputGenerationGuard {
private readonly latestGenerationByPty = new Map<string, number>()
private readonly incarnationByPty = new Map<string, string>()
private readonly sealedPtys = new Set<string>()
private readonly closedGenerations = new SshPtyClosedGenerationRanges()
constructor(private readonly isDisposed: () => boolean) {}
validateData(event: SshPtyOutputDataEvent): void {
if (
!event.id ||
!event.ptyIncarnation ||
!Number.isSafeInteger(event.providerGeneration) ||
event.providerGeneration <= 0 ||
!validOutputLength(event.rawLength) ||
(event.source !== undefined &&
(!event.source.spanId ||
!event.source.deliveryToken ||
!Number.isSafeInteger(event.source.clientGeneration) ||
event.source.clientGeneration <= 0 ||
!Number.isSafeInteger(event.source.ownerGeneration) ||
event.source.ownerGeneration <= 0 ||
!Number.isSafeInteger(event.source.sourceStartSu) ||
event.source.sourceStartSu < 0 ||
!Number.isSafeInteger(event.source.sourceEndSu) ||
event.source.sourceEndSu - event.source.sourceStartSu !== event.rawLength))
) {
throw outputIntakeError('ssh_output_invalid_event')
}
this.validate(event)
if (this.sealedPtys.has(sshPtyGenerationKey(event.id, event.providerGeneration))) {
throw outputIntakeError('ssh_output_after_exit')
}
}
sealExit(event: SshPtyOutputExitEvent): void {
this.validate(event)
const key = sshPtyGenerationKey(event.id, event.providerGeneration)
if (this.sealedPtys.has(key)) {
throw outputIntakeError('ssh_output_duplicate_exit')
}
this.sealedPtys.add(key)
}
validate(event: { id: string; providerGeneration: number; ptyIncarnation: string }): void {
if (this.isDisposed()) {
throw outputIntakeError('ssh_output_intake_disposed')
}
if (this.closedGenerations.has(event.providerGeneration)) {
throw outputIntakeError('ssh_output_stale_generation')
}
const generation = this.latestGenerationByPty.get(event.id)
if (generation !== undefined && event.providerGeneration < generation) {
throw outputIntakeError('ssh_output_stale_generation')
}
const incarnation = this.incarnationByPty.get(event.id)
if (
generation === event.providerGeneration &&
incarnation !== undefined &&
incarnation !== event.ptyIncarnation
) {
throw outputIntakeError('ssh_output_stale_incarnation')
}
if (generation === undefined || event.providerGeneration > generation) {
this.latestGenerationByPty.set(event.id, event.providerGeneration)
this.incarnationByPty.set(event.id, event.ptyIncarnation)
}
}
closeGeneration(providerGeneration: number): void {
this.closedGenerations.add(providerGeneration)
for (const [ptyId, generation] of this.latestGenerationByPty) {
if (generation === providerGeneration) {
this.latestGenerationByPty.delete(ptyId)
this.incarnationByPty.delete(ptyId)
this.sealedPtys.delete(sshPtyGenerationKey(ptyId, generation))
}
}
const prefix = `${providerGeneration}\0`
for (const key of this.sealedPtys) {
if (key.startsWith(prefix)) {
this.sealedPtys.delete(key)
}
}
}
activeGenerations(): ReadonlySet<number> {
return new Set(this.latestGenerationByPty.values())
}
getDebugSnapshot() {
return {
closedRanges: this.closedGenerations.size,
activeGaps: this.closedGenerations.activeGaps,
activePtys: this.latestGenerationByPty.size,
sealedPtys: this.sealedPtys.size
}
}
}
@@ -0,0 +1,77 @@
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
import type {
SshPtyModelAdmissionOptions,
SshPtyModelAdmissionReceipt
} from './ssh-pty-model-admission-contract'
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
export type SshPtySourceCancellationRequest = Readonly<{
id: string
clientGeneration: number
ownerGeneration: number
deliveryToken: string
}>
export type SshPtySourceCancellationProof = Readonly<{
sentEndSu: number
creditedEndSu: number
}>
export type SshPtyOutputDataEvent = Readonly<{
id: string
data: string
providerGeneration: number
ptyIncarnation: string
rawLength: number
transformed: boolean
sequence?: number
source?: Readonly<{
relayPtyId?: string
spanId: string
clientGeneration: number
ownerGeneration: number
deliveryToken: string
sourceStartSu: number
sourceEndSu: number
}>
}>
export type SshPtyOutputExitEvent = Readonly<{
id: string
code: number
providerGeneration: number
ptyIncarnation: string
}>
export type SshPtyOutputReceipt = SshPtyModelAdmissionReceipt &
Readonly<{ projection: LegacySshProjectionSemantics }>
export type SshPtyOutputIntakeDependencies = {
getModelSequence: (id: string) => number
acceptModel: (
event: SshPtyOutputDataEvent,
projection: LegacySshProjectionSemantics
) => { sequence: number; completion: Promise<void> }
project: (event: SshPtyOutputDataEvent, projection: LegacySshProjectionSemantics) => void
prepareExit: (event: SshPtyOutputExitEvent) => void | (() => void)
finalizeExit: (event: SshPtyOutputExitEvent) => void
pauseProvider?: (providerGeneration: number, id: string) => boolean
resumeProvider?: (providerGeneration: number, id: string) => void
closeProvider?: (providerGeneration: number, reason: string) => void
resetModelForMigration?: (providerGeneration: number, id: string) => void
onGenerationClosed?: (providerGeneration: number, reason: string) => void
publishSourceAck?: (
providerGeneration: number,
batch: PtySourceCreditAckBatch,
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
) => void
cancelSourceDelivery?: (
providerGeneration: number,
request: SshPtySourceCancellationRequest
) => Promise<SshPtySourceCancellationProof>
}
export type SshPtyOutputIntakeOptions = SshPtyModelAdmissionOptions & {
exitBarrierMs?: number
exitCancellationProofMs?: number
}
@@ -0,0 +1,151 @@
import type { SshPtyOutputIntake } from './ssh-pty-output-intake'
import type {
SshPtyOutputDataEvent,
SshPtyOutputExitEvent,
SshPtyOutputReceipt,
SshPtySourceCancellationProof,
SshPtySourceCancellationRequest
} from './ssh-pty-output-intake-contract'
import type { SshPtyAcceptedSourceCheckpoint } from './ssh-pty-output-source-obligations'
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
import type { SshPtyOutputGenerationMigration } from './ssh-pty-output-model-migration'
let installedIntake: SshPtyOutputIntake | null = null
let nextProviderGeneration = 1
const sourceAckPublishers = new Map<number, SshPtySourceAckPublisher>()
const sourceCancellationPublishers = new Map<number, SshPtySourceCancellationPublisher>()
type SshPtySourceAckPublisher = (
batch: PtySourceCreditAckBatch,
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
) => void
type SshPtySourceCancellationPublisher = (
request: SshPtySourceCancellationRequest
) => Promise<SshPtySourceCancellationProof>
export function allocateSshPtyProviderGeneration(): number {
return nextProviderGeneration++
}
export function installSshPtyOutputIntake(intake: SshPtyOutputIntake): () => void {
const previous = installedIntake
installedIntake = intake
previous?.dispose()
return () => {
if (installedIntake === intake) {
installedIntake = null
intake.dispose()
}
}
}
export function acceptSshPtyOutputData(event: SshPtyOutputDataEvent): Promise<SshPtyOutputReceipt> {
return installedIntake
? installedIntake.acceptData(event)
: Promise.reject(outputIntakeUnavailableError())
}
export function acceptSshPtyOutputExit(event: SshPtyOutputExitEvent): Promise<void> {
return installedIntake
? installedIntake.acceptExit(event)
: Promise.reject(outputIntakeUnavailableError())
}
export function closeSshPtyOutputGeneration(providerGeneration: number, reason: string): void {
installedIntake?.closeGeneration(providerGeneration, reason)
}
export function getSshPtyAcceptedSourceCheckpoints(
providerGeneration: number
): readonly SshPtyAcceptedSourceCheckpoint[] {
return installedIntake?.getAcceptedSourceCheckpoints(providerGeneration) ?? []
}
export function beginSshPtyOutputGenerationMigration(
providerGeneration: number
): SshPtyOutputGenerationMigration {
return (
installedIntake?.beginGenerationMigration(providerGeneration) ?? {
byPty: new Map(),
completion: Promise.resolve()
}
)
}
export function installSshPtySourceAckPublisher(
providerGeneration: number,
publish: SshPtySourceAckPublisher
): () => void {
if (sourceAckPublishers.has(providerGeneration)) {
throw new Error('ssh_source_ack_publisher_duplicate_generation')
}
sourceAckPublishers.set(providerGeneration, publish)
return () => {
if (sourceAckPublishers.get(providerGeneration) === publish) {
sourceAckPublishers.delete(providerGeneration)
}
}
}
export function publishSshPtySourceAck(
providerGeneration: number,
batch: PtySourceCreditAckBatch,
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
): void {
const publisher = sourceAckPublishers.get(providerGeneration)
if (!publisher) {
onSettled({ ok: false, error: new Error('ssh_source_ack_publisher_unavailable') })
return
}
publisher(batch, onSettled)
}
export function installSshPtySourceCancellationPublisher(
providerGeneration: number,
cancel: SshPtySourceCancellationPublisher
): () => void {
if (sourceCancellationPublishers.has(providerGeneration)) {
throw new Error('ssh_source_cancellation_publisher_duplicate_generation')
}
sourceCancellationPublishers.set(providerGeneration, cancel)
return () => {
if (sourceCancellationPublishers.get(providerGeneration) === cancel) {
sourceCancellationPublishers.delete(providerGeneration)
}
}
}
export function cancelSshPtySourceDelivery(
providerGeneration: number,
request: SshPtySourceCancellationRequest
): Promise<SshPtySourceCancellationProof> {
const publisher = sourceCancellationPublishers.get(providerGeneration)
return publisher
? publisher(request)
: Promise.reject(new Error('ssh_source_cancellation_publisher_unavailable'))
}
export function applySshPtySourceCancellationProof(
event: SshPtyOutputExitEvent,
proof: SshPtySourceCancellationProof
): boolean {
return installedIntake?.applySourceCancellationProof(event, proof) ?? false
}
export function applySshPtySourceRecoveryCancellationProof(
event: SshPtyOutputExitEvent,
proof: SshPtySourceCancellationProof
): boolean {
if (!installedIntake) {
return false
}
installedIntake.applySourceRecoveryCancellationProof(event, proof)
return true
}
function outputIntakeUnavailableError(): Error {
return Object.assign(new Error('ssh_output_intake_unavailable'), {
code: 'ssh_output_intake_unavailable'
})
}
@@ -0,0 +1,62 @@
import { vi } from 'vitest'
import {
SshPtyOutputIntake,
type SshPtyOutputDataEvent,
type SshPtyOutputIntakeDependencies
} from './ssh-pty-output-intake'
export function sshPtyOutputEvent(
overrides: Partial<SshPtyOutputDataEvent> = {}
): SshPtyOutputDataEvent {
return {
id: 'pty-1',
data: 'aaaa',
providerGeneration: 1,
ptyIncarnation: 'incarnation-1',
rawLength: 4,
transformed: false,
...overrides
}
}
export function createSshPtyOutputIntakeHarness(
overrides: Partial<SshPtyOutputIntakeDependencies> = {},
options: ConstructorParameters<typeof SshPtyOutputIntake>[1] = {}
) {
let sequence = 0
const completions: ReturnType<typeof deferred>[] = []
const order: string[] = []
const dependencies: SshPtyOutputIntakeDependencies = {
getModelSequence: () => sequence,
acceptModel: (input) => {
order.push(`model:${input.data}`)
sequence += input.rawLength
const completion = deferred()
completions.push(completion)
return { sequence, completion: completion.promise }
},
project: (input) => order.push(`project:${input.data}`),
prepareExit: vi.fn(),
finalizeExit: () => order.push('exit'),
pauseProvider: vi.fn(() => true),
resumeProvider: vi.fn(),
closeProvider: vi.fn(),
...overrides
}
return {
intake: new SshPtyOutputIntake(dependencies, options),
dependencies,
completions,
order
}
}
function deferred() {
let resolve!: () => void
let reject!: (error: Error) => void
const promise = new Promise<void>((promiseResolve, promiseReject) => {
resolve = promiseResolve
reject = promiseReject
})
return { promise, resolve, reject }
}
@@ -0,0 +1,16 @@
export type SshPtyExitBarrier = {
timer: ReturnType<typeof setTimeout>
reject: (error: Error) => void
}
export function outputIntakeError(code: string): Error {
return Object.assign(new Error(code), { code })
}
export function validOutputLength(value: number): boolean {
return Number.isSafeInteger(value) && value >= 0
}
export function sshPtyGenerationKey(ptyId: string, providerGeneration: number): string {
return `${providerGeneration}\0${ptyId}`
}
+760
View File
@@ -0,0 +1,760 @@
import { describe, expect, it, vi } from 'vitest'
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
import {
createSshPtyOutputIntakeHarness as createHarness,
sshPtyOutputEvent as event
} from './ssh-pty-output-intake-test-harness'
describe('SshPtyOutputIntake', () => {
it('plateaus at the model and pressure budgets, then resumes below low water', async () => {
const harness = createHarness(
{},
{
perPtyHighSourceUnits: 4,
perPtyHighBytes: 1024,
perPtyLowSourceUnits: 1,
perPtyLowBytes: 256,
globalHighSourceUnits: 4,
globalHighBytes: 1024,
globalLowSourceUnits: 1,
globalLowBytes: 256,
pressureMaxFrames: 2,
pressureMaxBytes: 1024
}
)
const first = harness.intake.acceptData(event())
const second = harness.intake.acceptData(event({ data: 'bbbb' }))
const third = harness.intake.acceptData(event({ data: 'cccc' }))
const rejected = harness.intake.acceptData(event({ data: 'dddd' }))
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
sourceUnits: 4,
pressureFrames: 2
})
await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted')
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, expect.any(String))
harness.completions[0]!.resolve()
await first
harness.completions[1]!.resolve()
await second
harness.completions[2]!.resolve()
await third
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
sourceUnits: 0,
bytes: 0,
pressureFrames: 0
})
expect(harness.dependencies.resumeProvider).toHaveBeenCalled()
})
it('preserves per-PTY FIFO while differently sized pressure entries wait', async () => {
const harness = createHarness(
{},
{
perPtyHighSourceUnits: 4,
perPtyHighBytes: 4096,
globalHighSourceUnits: 4,
globalHighBytes: 4096,
pressureMaxFrames: 4,
pressureMaxBytes: 4096
}
)
const receipts = [
harness.intake.acceptData(event({ data: 'a', rawLength: 1 })),
harness.intake.acceptData(event({ data: 'bbb', rawLength: 3 })),
harness.intake.acceptData(event({ data: 'cc', rawLength: 2 })),
harness.intake.acceptData(event({ data: 'd', rawLength: 1 }))
]
harness.completions[0]!.resolve()
await receipts[0]
harness.completions[1]!.resolve()
await receipts[1]
harness.completions[2]!.resolve()
await receipts[2]
harness.completions[3]!.resolve()
await receipts[3]
expect(harness.order).toEqual([
'model:a',
'project:a',
'model:bbb',
'project:bbb',
'model:cc',
'project:cc',
'model:d',
'project:d'
])
})
it('assigns projection sequence ends when queued model capture begins', async () => {
const harness = createHarness(
{},
{
perPtyHighSourceUnits: 12,
perPtyHighBytes: 4096,
globalHighSourceUnits: 12,
globalHighBytes: 4096
}
)
const first = harness.intake.acceptData(event({ data: 'aaaa' }))
const second = harness.intake.acceptData(event({ data: 'bbbb' }))
const third = harness.intake.acceptData(event({ data: 'cccc' }))
harness.completions[0]!.resolve()
expect((await first).projection.identity.sequenceEnd).toBe(4)
harness.completions[1]!.resolve()
expect((await second).projection.identity.sequenceEnd).toBe(8)
harness.completions[2]!.resolve()
expect((await third).projection.identity.sequenceEnd).toBe(12)
})
it('transfers projection state and closes the provider on model failure', async () => {
const harness = createHarness()
const receipt = harness.intake.acceptData(event())
harness.completions[0]!.reject(new Error('emulator failed'))
await expect(receipt).rejects.toThrow('emulator failed')
expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1)
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed')
})
it('transfers a committed projection when desktop admission throws', async () => {
const projections: LegacySshProjectionSemantics[] = []
const harness = createHarness({
project: (_event, projection) => {
projections.push(projection)
if (projections.length === 1) {
throw new Error('send failed')
}
}
})
const receipt = harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5 }))
harness.completions[0]!.resolve()
await expect(receipt).resolves.toMatchObject({ sequence: 5 })
expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1)
const next = harness.intake.acceptData(event({ data: '31h', rawLength: 3 }))
harness.completions[1]!.resolve()
await next
expect(projections[1]).toMatchObject({
identity: { displayStart: 5 },
beforeScanner: { tail: '\x1b[?20', pendingSubscribe: false },
decision: 'subscribed'
})
})
it('commits an immutable desktop source identity through projection admission', async () => {
const projections: LegacySshProjectionSemantics[] = []
const harness = createHarness({
project: (_event, projection) => projections.push(projection)
})
const receipt = harness.intake.acceptData(
event({
source: {
spanId: 'span-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 10,
sourceEndSu: 14
}
})
)
harness.completions[0]!.resolve()
await receipt
expect(projections[0]?.desktopSpan).toMatchObject({
spanId: 'span-1',
projectionSemanticsId: projections[0]?.identity.projectionSemanticsId,
providerGeneration: 1,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
sourceStartSu: 10,
sourceEndSu: 14,
displayStart: 0,
displayEnd: 4,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
})
expect(Object.isFrozen(projections[0]?.desktopSpan)).toBe(true)
})
it('exports only the model-settled source boundary before generation close', async () => {
const harness = createHarness()
const receipt = harness.intake.acceptData(
event({
source: {
spanId: 'recovery-span-1',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'delivery-token-1',
sourceStartSu: 10,
sourceEndSu: 14
}
})
)
expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(10)
harness.completions[0]!.resolve()
await receipt
expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual([
{
id: 'pty-1',
providerGeneration: 1,
clientGeneration: 3,
ownerGeneration: 4,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'delivery-token-1',
acceptedSourceEndSu: 14
}
])
})
it('rolls back projection staging when model capture throws synchronously', async () => {
const project = vi.fn()
const harness = createHarness({
acceptModel: () => {
throw new Error('model reservation failed')
},
project
})
await expect(harness.intake.acceptData(event())).rejects.toThrow('model reservation failed')
expect(project).not.toHaveBeenCalled()
expect(harness.intake.getDebugSnapshot().projection).toMatchObject({
rolledBack: 1,
records: 0
})
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed')
})
it('rolls back projection staging when source reservation validation fails', async () => {
const harness = createHarness()
const source = {
spanId: 'duplicate',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
const first = harness.intake.acceptData(event({ source }))
harness.completions[0]!.resolve()
await first
await expect(
harness.intake.acceptData(
event({
source: { ...source, sourceStartSu: 4, sourceEndSu: 8 }
})
)
).rejects.toThrow('duplicate')
expect(harness.intake.getDebugSnapshot().projection).toMatchObject({
rolledBack: 1,
records: 1
})
})
it('rolls back committed source and scanner facts when model capture throws', async () => {
let attempts = 0
const harness = createHarness({
acceptModel: (accepted) => {
attempts++
if (attempts === 1) {
throw new Error('model reservation failed')
}
return {
sequence: accepted.rawLength,
completion: Promise.resolve()
}
}
})
const source = {
spanId: 'span-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 5
}
await expect(
harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5, source }))
).rejects.toThrow('model reservation failed')
const retry = await harness.intake.acceptData(
event({ data: '\x1b[?20', rawLength: 5, source: { ...source, spanId: 'span-2' } })
)
expect(retry.projection.identity.displayStart).toBe(0)
expect(retry.projection.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
})
it('rejects stale provider generations without model capture', async () => {
const harness = createHarness()
const current = harness.intake.acceptData(event({ providerGeneration: 2 }))
harness.completions[0]!.resolve()
await current
await expect(harness.intake.acceptData(event({ providerGeneration: 1 }))).rejects.toThrow(
'ssh_output_stale_generation'
)
expect(harness.completions).toHaveLength(1)
})
it('keeps exit behind accepted model and projection work', async () => {
const harness = createHarness({}, { exitBarrierMs: 1000 })
const dataReceipt = harness.intake.acceptData(event())
const exitReceipt = harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
await Promise.resolve()
expect(harness.order).toEqual(['model:aaaa', 'project:aaaa'])
harness.completions[0]!.resolve()
await Promise.all([dataReceipt, exitReceipt])
expect(harness.order).toEqual(['model:aaaa', 'project:aaaa', 'exit'])
})
it('admits queued pre-exit source spans before sealing the token', async () => {
const harness = createHarness({}, { exitBarrierMs: 1000 })
const first = harness.intake.acceptData(
event({
source: {
spanId: 'span-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
const second = harness.intake.acceptData(
event({
data: 'bbbb',
source: {
spanId: 'span-2',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 4,
sourceEndSu: 8
}
})
)
const exit = harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
harness.completions[0]!.resolve()
await first
harness.completions[1]!.resolve()
await Promise.all([second, exit])
expect(harness.order).toEqual([
'model:aaaa',
'project:aaaa',
'model:bbbb',
'project:bbbb',
'exit'
])
})
it('does not pump queued model work after a running completion fails', async () => {
const harness = createHarness()
const first = harness.intake.acceptData(event({ data: 'first' }))
const queued = harness.intake.acceptData(event({ data: 'queued' }))
harness.completions[0]!.reject(new Error('emulator failed'))
await expect(first).rejects.toThrow('emulator failed')
await expect(queued).rejects.toThrow('ssh_model_admission_completion_failed')
expect(harness.order).toEqual(['model:first', 'project:first'])
expect(harness.completions).toHaveLength(1)
})
it('retains exited delivery until published renderer projections settle', async () => {
const harness = createHarness({}, { exitBarrierMs: 1000 })
const dataReceipt = harness.intake.acceptData(event())
harness.completions[0]!.resolve()
const receipt = await dataReceipt
harness.intake.publishProjectionPrefix(
[receipt.projection.identity.projectionSemanticsId],
4,
4
)
let exited = false
const exitReceipt = harness.intake
.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
.then(() => {
exited = true
})
await Promise.resolve()
expect(exited).toBe(false)
expect(harness.intake.getDebugSnapshot().projection.records).toBe(1)
expect(harness.intake.settleProjectionPrefix('pty-1', 4)).toBe(4)
await exitReceipt
expect(harness.order.at(-1)).toBe('exit')
expect(harness.intake.getDebugSnapshot().projection.records).toBe(0)
})
it('owns renderer exit preparation through finalization and duplicate rejection', async () => {
const releaseRendererExit = vi.fn()
const harness = createHarness({
prepareExit: vi.fn(() => releaseRendererExit)
})
const dataReceipt = harness.intake.acceptData(event())
harness.completions[0]!.resolve()
const receipt = await dataReceipt
harness.intake.publishProjectionPrefix(
[receipt.projection.identity.projectionSemanticsId],
4,
4
)
const exitEvent = {
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
}
const exit = harness.intake.acceptExit(exitEvent)
await Promise.resolve()
expect(releaseRendererExit).not.toHaveBeenCalled()
await expect(harness.intake.acceptExit(exitEvent)).rejects.toThrow('ssh_output_duplicate_exit')
expect(releaseRendererExit).not.toHaveBeenCalled()
harness.intake.settleProjectionPrefix('pty-1', 4)
await exit
expect(releaseRendererExit).toHaveBeenCalledOnce()
})
it('releases renderer exit preparation when generation close aborts finalization', async () => {
const releaseRendererExit = vi.fn()
const harness = createHarness({
prepareExit: vi.fn(() => releaseRendererExit)
})
const dataReceipt = harness.intake.acceptData(event())
harness.completions[0]!.resolve()
const receipt = await dataReceipt
harness.intake.publishProjectionPrefix(
[receipt.projection.identity.projectionSemanticsId],
4,
4
)
const exit = harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
await Promise.resolve()
harness.intake.closeGeneration(1, 'provider-replaced')
await expect(exit).rejects.toThrow('provider-replaced')
expect(releaseRendererExit).toHaveBeenCalledOnce()
expect(harness.order).not.toContain('exit')
})
it('retains exit until a required remote source consumer settles', async () => {
const harness = createHarness({}, { exitBarrierMs: 1000 })
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
const stream = {
ptyId: 'pty-1',
consumerId: 'remote-1',
streamGeneration: 'stream-1'
}
expect(remote.attach(stream)).toBe(true)
const dataReceipt = harness.intake.acceptData(
event({
source: {
spanId: 'span-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
harness.completions[0]!.resolve()
const receipt = await dataReceipt
let exited = false
const exitReceipt = harness.intake
.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
.then(() => {
exited = true
})
await Promise.resolve()
expect(exited).toBe(false)
remote.settle(stream, [receipt.projection.desktopSpan!])
await exitReceipt
expect(harness.order.at(-1)).toBe('exit')
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
})
it('keeps timed-out exit projections until generation-close proof', async () => {
const harness = createHarness(
{
cancelSourceDelivery: () => Promise.reject(new Error('cancel transport failed'))
},
{ exitBarrierMs: 1, exitCancellationProofMs: 10 }
)
const dataReceipt = harness.intake.acceptData(event())
harness.completions[0]!.resolve()
const receipt = await dataReceipt
harness.intake.publishProjectionPrefix(
[receipt.projection.identity.projectionSemanticsId],
4,
4
)
await expect(
harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
).rejects.toThrow('ssh_source_cancellation_identity_unavailable')
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(
1,
'ssh-exit-cancellation-proof-failed'
)
expect(harness.intake.getDebugSnapshot().projection.records).toBe(1)
harness.intake.closeGeneration(1, 'provider-closed')
expect(harness.intake.getDebugSnapshot().projection.records).toBe(0)
})
it('cancels only the timed-out source delivery and keeps the provider usable', async () => {
const cancelSourceDelivery = vi.fn(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
const harness = createHarness(
{ cancelSourceDelivery },
{ exitBarrierMs: 1, exitCancellationProofMs: 100 }
)
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
const stream = {
ptyId: 'pty-1',
consumerId: 'remote-1',
streamGeneration: 'stream-1'
}
remote.attach(stream)
const dataReceipt = harness.intake.acceptData(
event({
source: {
spanId: 'span-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
harness.completions[0]!.resolve()
await dataReceipt
await harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
expect(cancelSourceDelivery).toHaveBeenCalledWith(1, {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1'
})
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
expect(harness.order.at(-1)).toBe('exit')
const sibling = harness.intake.acceptData(
event({ id: 'pty-2', ptyIncarnation: 'incarnation-2' })
)
harness.completions[1]!.resolve()
await expect(sibling).resolves.toMatchObject({ ptyId: 'pty-2' })
})
it('accepts recovery cancellation proof before any replacement span is admitted', () => {
const harness = createHarness()
expect(() =>
harness.intake.applySourceRecoveryCancellationProof(
{
id: 'pty-1',
code: -1,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
},
{ sentEndSu: 8, creditedEndSu: 4 }
)
).not.toThrow()
expect(harness.intake.getDebugSnapshot().source).toEqual({
openedTokens: 0,
ptyIdentities: 0
})
})
it('reclaims a partially admitted recovery prefix from authoritative proof', async () => {
const harness = createHarness()
const receipt = harness.intake.acceptData(
event({
source: {
spanId: 'recovery-span',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'recovery-token',
sourceStartSu: 4,
sourceEndSu: 8
}
})
)
harness.completions[0]!.resolve()
await receipt
harness.intake.applySourceRecoveryCancellationProof(
{
id: 'pty-1',
code: -1,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
},
{ sentEndSu: 12, creditedEndSu: 4 }
)
expect(harness.intake.getDebugSnapshot().source).toEqual({
openedTokens: 0,
ptyIdentities: 0
})
})
it('rejects late same-generation data after ordered exit cleanup', async () => {
const harness = createHarness({}, { exitBarrierMs: 1000 })
const first = harness.intake.acceptData(event())
harness.completions[0]!.resolve()
await first
await harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_after_exit')
await expect(
harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
).rejects.toThrow('ssh_output_duplicate_exit')
const next = harness.intake.acceptData(
event({ providerGeneration: 2, ptyIncarnation: 'incarnation-2' })
)
harness.completions[1]!.resolve()
await expect(next).resolves.toMatchObject({
projection: { identity: { ptyIncarnation: 'incarnation-2', displayStart: 0 } }
})
})
it('closes the provider when exit finalization fails', async () => {
const releaseRendererExit = vi.fn()
const harness = createHarness({
prepareExit: () => releaseRendererExit,
finalizeExit: () => {
throw new Error('renderer exit send failed')
}
})
await expect(
harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
).rejects.toThrow('renderer exit send failed')
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'pty-exit-finalize-failed')
expect(releaseRendererExit).toHaveBeenCalledOnce()
})
it('cancels queued work and exit waiters on generation cleanup', async () => {
const harness = createHarness(
{},
{
perPtyHighSourceUnits: 8,
perPtyHighBytes: 2048,
globalHighSourceUnits: 8,
globalHighBytes: 2048
}
)
const running = harness.intake.acceptData(event())
const queued = harness.intake.acceptData(event({ data: 'bbbb' }))
const exit = harness.intake.acceptExit({
id: 'pty-1',
code: 0,
providerGeneration: 1,
ptyIncarnation: 'incarnation-1'
})
harness.intake.closeGeneration(1, 'provider-closed')
await expect(queued).rejects.toThrow('provider-closed')
await expect(exit).rejects.toThrow('provider-closed')
await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_stale_generation')
await expect(running).rejects.toThrow('provider-closed')
expect(harness.intake.getDebugSnapshot().model).toMatchObject({ sourceUnits: 0, bytes: 0 })
harness.completions[0]!.resolve()
await Promise.resolve()
expect(harness.intake.getDebugSnapshot().exitBarriers).toBe(0)
})
it('resumes a paused provider during generation cleanup', async () => {
const harness = createHarness(
{},
{
perPtyHighSourceUnits: 4,
perPtyHighBytes: 1024,
globalHighSourceUnits: 4,
globalHighBytes: 1024,
pressureMaxFrames: 2,
pressureMaxBytes: 1024
}
)
const running = harness.intake.acceptData(event())
const pressured = harness.intake.acceptData(event({ data: 'bbbb' }))
harness.intake.closeGeneration(1, 'provider-closed')
await expect(pressured).rejects.toThrow('provider-closed')
expect(harness.dependencies.resumeProvider).toHaveBeenCalledWith(1, 'pty-1')
await expect(running).rejects.toThrow('provider-closed')
harness.completions[0]!.resolve()
await Promise.resolve()
})
})
+296
View File
@@ -0,0 +1,296 @@
import {
SshPtyLegacyProjectionLedger,
type LegacySshProjectionSemantics
} from './ssh-pty-legacy-projection'
import { SshPtyModelAdmission } from './ssh-pty-model-admission'
import { SshPtyOutputExitDeadline } from './ssh-pty-output-exit-deadline'
import { settleSshPtyOutputExit } from './ssh-pty-output-exit'
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
import {
SshPtyOutputModelMigration,
type SshPtyOutputGenerationMigration,
type SshPtyTrackedModelAdmission
} from './ssh-pty-output-model-migration'
import {
SshPtyOutputSourceObligations,
type SshPtyOutputSourceReservation
} from './ssh-pty-output-source-obligations'
import type {
SshPtyOutputDataEvent,
SshPtyOutputExitEvent,
SshPtyOutputIntakeDependencies,
SshPtyOutputIntakeOptions,
SshPtyOutputReceipt
} from './ssh-pty-output-intake-contract'
import { outputIntakeError } from './ssh-pty-output-intake-validation'
export type {
SshPtyOutputDataEvent,
SshPtyOutputExitEvent,
SshPtyOutputIntakeDependencies,
SshPtyOutputIntakeOptions,
SshPtyOutputReceipt
} from './ssh-pty-output-intake-contract'
export class SshPtyOutputIntake {
private readonly projections: SshPtyLegacyProjectionLedger
private readonly sourceObligations: SshPtyOutputSourceObligations
private readonly generationGuard = new SshPtyOutputGenerationGuard(() => this.disposed)
private readonly admission: SshPtyModelAdmission
private readonly modelMigration: SshPtyOutputModelMigration
private readonly exitDeadline: SshPtyOutputExitDeadline
private disposed = false
constructor(
private readonly dependencies: SshPtyOutputIntakeDependencies,
options: SshPtyOutputIntakeOptions = {}
) {
this.sourceObligations = new SshPtyOutputSourceObligations(dependencies.publishSourceAck)
this.projections = new SshPtyLegacyProjectionLedger({
onSettled: (span) => this.sourceObligations.settleDesktop(span, 'renderer-parse'),
onTransferred: (span, reason) => this.sourceObligations.transferDesktop(span, reason)
})
this.admission = new SshPtyModelAdmission({
...options,
pauseProvider: (key) =>
this.dependencies.pauseProvider?.(key.providerGeneration, key.ptyId) ?? false,
resumeProvider: (key) =>
this.dependencies.resumeProvider?.(key.providerGeneration, key.ptyId),
closeProvider: (providerGeneration, reason) =>
this.dependencies.closeProvider?.(providerGeneration, reason)
})
this.modelMigration = new SshPtyOutputModelMigration(
this.admission,
this.sourceObligations,
(providerGeneration, ptyId) =>
this.dependencies.resetModelForMigration?.(providerGeneration, ptyId)
)
this.exitDeadline = new SshPtyOutputExitDeadline({
admission: this.admission,
projections: this.projections,
sourceObligations: this.sourceObligations,
intake: this.dependencies,
barrierMs: options.exitBarrierMs,
cancellationProofMs: options.exitCancellationProofMs
})
}
acceptData(event: SshPtyOutputDataEvent): Promise<SshPtyOutputReceipt> {
try {
this.generationGuard.validateData(event)
} catch (error) {
return Promise.reject(error)
}
let projection: LegacySshProjectionSemantics | undefined
let sourceReservation: SshPtyOutputSourceReservation | undefined
const key = { ptyId: event.id, providerGeneration: event.providerGeneration }
const tracked: SshPtyTrackedModelAdmission = { key, started: false }
const receipt = this.admission.accept(key, event.data, event.rawLength, () => {
tracked.started = true
const expectedSequence = this.dependencies.getModelSequence(event.id) + event.rawLength
const reservation = this.projections.reserve({
ptyId: event.id,
providerGeneration: event.providerGeneration,
ptyIncarnation: event.ptyIncarnation,
data: event.data,
sequenceEnd: expectedSequence,
rawLength: event.rawLength,
transformed: event.transformed,
source: event.source
})
try {
if (reservation.semantics.desktopSpan) {
sourceReservation = this.sourceObligations.reserve(
event,
reservation.semantics.desktopSpan
)
}
} catch (error) {
this.projections.rollback(reservation)
throw error
}
try {
projection = this.projections.commit(reservation)
if (sourceReservation) {
this.sourceObligations.commit(
sourceReservation,
event.id,
projection.identity.sequenceEnd
)
}
} catch (error) {
if (sourceReservation) {
this.sourceObligations.rollback(sourceReservation)
}
if (!this.projections.rollbackCommitted(reservation)) {
this.projections.rollback(reservation)
}
throw error
}
let model: { sequence: number; completion: Promise<void> }
try {
model = this.dependencies.acceptModel(event, projection)
} catch (error) {
if (sourceReservation) {
this.sourceObligations.rollback(sourceReservation)
}
this.projections.rollbackCommitted(reservation)
throw error
}
try {
this.dependencies.project(event, projection)
} catch {
const id = projection.identity.projectionSemanticsId
this.projections.transfer([id], 'projection-admission-failed')
}
return model
})
const completion = receipt.then(
(modelReceipt) => {
if (!projection) {
throw outputIntakeError('ssh_projection_receipt_missing')
}
if (sourceReservation) {
this.sourceObligations.settleModel(sourceReservation.span)
}
return Object.freeze({ ...modelReceipt, projection })
},
(error) => {
if (projection) {
this.projections.transfer(
[projection.identity.projectionSemanticsId],
'model-admission-failed'
)
}
const code = (error as { code?: unknown }).code
if (
code !== 'ssh_exit_delivery_canceled' &&
!(typeof code === 'string' && code.startsWith('ssh_model_migration_'))
) {
this.dependencies.closeProvider?.(event.providerGeneration, 'model-admission-failed')
}
throw error
}
)
tracked.completion = completion
this.modelMigration.track(tracked)
return completion
}
async acceptExit(event: SshPtyOutputExitEvent): Promise<void> {
this.generationGuard.sealExit(event)
await this.exitDeadline.wait(event, (validateNormalExit) =>
this.finishExit(event, validateNormalExit)
)
}
private async finishExit(
event: SshPtyOutputExitEvent,
validateNormalExit: () => void
): Promise<void> {
await settleSshPtyOutputExit({
event,
admission: this.admission,
projections: this.projections,
dependencies: this.dependencies,
validateGeneration: () => {
this.generationGuard.validate(event)
validateNormalExit()
},
prepareExit: () => this.exitDeadline.prepareExitOnce(event),
afterAdmissionIdle: () => this.sourceObligations.sealPty(event),
waitForSourceTerminal: () => this.sourceObligations.whenPtyTerminal(event),
beforeFinalize: () => this.sourceObligations.markExitPublished(event)
})
}
publishProjectionPrefix(
ids: readonly string[],
displayChars: number,
accountingChars: number
): void {
this.projections.publishPrefix(ids, displayChars, accountingChars)
}
settleProjectionPrefix(ptyId: string, accountingChars: number): number {
return this.projections.settlePublishedPrefix(ptyId, accountingChars)
}
transferProjections(ids: readonly string[], reason: string): number {
return this.projections.transfer(ids, reason)
}
transferPtyProjections(ptyId: string, reason: string): number {
return this.projections.transferPty(ptyId, reason)
}
hasProjectionFromGeneration(ids: readonly string[], providerGeneration: number): boolean {
return ids.some(
(id) => this.projections.get(id)?.identity.providerGeneration === providerGeneration
)
}
hasUnpublishedProjection(id: string): boolean {
return this.projections.hasUnpublished(id)
}
closeGeneration(providerGeneration: number, reason: string): void {
this.generationGuard.closeGeneration(providerGeneration)
this.admission.closeGeneration(providerGeneration, reason)
this.dependencies.onGenerationClosed?.(providerGeneration, reason)
this.projections.closeGeneration(providerGeneration, reason)
this.sourceObligations.closeGeneration(providerGeneration, reason)
this.exitDeadline.closeGeneration(providerGeneration, outputIntakeError(reason))
}
dispose(): void {
if (this.disposed) {
return
}
this.disposed = true
for (const generation of this.generationGuard.activeGenerations()) {
this.closeGeneration(generation, 'ssh_output_intake_disposed')
}
this.admission.dispose()
this.sourceObligations.dispose()
}
getRemoteSourceRangeConsumerHooks() {
return this.sourceObligations.remoteHooks
}
getAcceptedSourceCheckpoints(providerGeneration: number) {
return this.sourceObligations.acceptedCheckpoints(providerGeneration)
}
beginGenerationMigration(
providerGeneration: number,
timeoutMs?: number
): SshPtyOutputGenerationMigration {
return this.modelMigration.beginGeneration(providerGeneration, timeoutMs)
}
applySourceCancellationProof(
event: SshPtyOutputExitEvent,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): boolean {
return this.sourceObligations.applyCancellationProof(event, proof)
}
applySourceRecoveryCancellationProof(
event: SshPtyOutputExitEvent,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
this.sourceObligations.applyRecoveryCancellationProof(event, proof)
}
getDebugSnapshot() {
return {
model: this.admission.getDebugSnapshot(),
projection: this.projections.getDebugSnapshot(),
source: this.sourceObligations.getDebugSnapshot(),
generation: this.generationGuard.getDebugSnapshot(),
exitBarriers: this.exitDeadline.activeBarriers
}
}
}
@@ -0,0 +1,197 @@
import { describe, expect, it, vi } from 'vitest'
import {
createSshPtyOutputIntakeHarness as createHarness,
sshPtyOutputEvent as event
} from './ssh-pty-output-intake-test-harness'
describe('SshPtyOutputModelMigration', () => {
it('fences a running source span before exporting its migration checkpoint', async () => {
const harness = createHarness()
const first = harness.intake.acceptData(
event({
data: 'aaaa',
source: {
spanId: 'span-a',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'delivery-token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
harness.completions[0]!.resolve()
await first
const second = harness.intake.acceptData(
event({
data: 'bbbb',
source: {
spanId: 'span-b',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'delivery-token-1',
sourceStartSu: 4,
sourceEndSu: 8
}
})
)
const migration = harness.intake.beginGenerationMigration(1)
const result = migration.byPty.get('pty-1')
expect(result).toBeDefined()
await expect(Promise.race([result, Promise.resolve('pending')])).resolves.toBe('pending')
expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(4)
harness.completions[1]!.resolve()
await expect(second).resolves.toMatchObject({ sequence: 8 })
await expect(result).resolves.toMatchObject({
status: 'settled',
checkpoint: { acceptedSourceEndSu: 8 }
})
expect(harness.order.filter((entry) => entry === 'project:bbbb')).toHaveLength(1)
await migration.completion
})
it('times out one migration, resets its model, and releases retained admission once', async () => {
vi.useFakeTimers()
try {
const resetModelForMigration = vi.fn()
const harness = createHarness({ resetModelForMigration })
const receipt = harness.intake.acceptData(
event({
source: {
spanId: 'span-b',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'delivery-token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
const migration = harness.intake.beginGenerationMigration(1, 10_000)
const result = migration.byPty.get('pty-1')
await vi.advanceTimersByTimeAsync(10_000)
await expect(result).resolves.toMatchObject({ status: 'checkpoint-unavailable' })
await expect(receipt).rejects.toThrow('ssh_model_migration_timeout')
expect(resetModelForMigration).toHaveBeenCalledOnce()
expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-1')
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
sourceUnits: 0,
bytes: 0,
migratingPtys: 1
})
expect(vi.getTimerCount()).toBe(0)
harness.completions[0]!.resolve()
await Promise.resolve()
expect(resetModelForMigration).toHaveBeenCalledOnce()
harness.intake.closeGeneration(1, 'connection_lost')
expect(harness.intake.getDebugSnapshot().model.migratingPtys).toBe(0)
} finally {
vi.useRealTimers()
}
})
it('contains a running callback failure to its migrating PTY', async () => {
vi.useFakeTimers()
try {
const resetModelForMigration = vi.fn()
const harness = createHarness({ resetModelForMigration })
const sibling = harness.intake.acceptData(
event({
id: 'pty-sibling',
ptyIncarnation: 'incarnation-sibling',
source: {
spanId: 'span-sibling',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'delivery-sibling',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
harness.completions[0]!.resolve()
await sibling
const failed = harness.intake.acceptData(
event({
id: 'pty-failed',
ptyIncarnation: 'incarnation-failed',
source: {
spanId: 'span-failed',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'delivery-failed',
sourceStartSu: 0,
sourceEndSu: 4
}
})
)
const migration = harness.intake.beginGenerationMigration(1)
harness.completions[1]!.reject(new Error('emulator failed'))
await expect(failed).rejects.toMatchObject({
message: 'emulator failed',
code: 'ssh_model_migration_completion_failed'
})
await expect(migration.byPty.get('pty-failed')).resolves.toEqual({
status: 'checkpoint-unavailable',
reason: 'completion-failed'
})
await expect(migration.byPty.get('pty-sibling')).resolves.toMatchObject({
status: 'settled',
checkpoint: { id: 'pty-sibling', acceptedSourceEndSu: 4 }
})
await migration.completion
expect(resetModelForMigration).toHaveBeenCalledTimes(1)
expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-failed')
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual(
expect.arrayContaining([
expect.objectContaining({ id: 'pty-failed', acceptedSourceEndSu: 0 }),
expect.objectContaining({ id: 'pty-sibling', acceptedSourceEndSu: 4 })
])
)
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
sourceUnits: 0,
bytes: 0,
pressureFrames: 0,
migratingPtys: 2
})
expect(vi.getTimerCount()).toBe(0)
harness.completions[1]!.resolve()
await Promise.resolve()
expect(resetModelForMigration).toHaveBeenCalledTimes(1)
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
expect(
harness.intake
.getAcceptedSourceCheckpoints(1)
.find((checkpoint) => checkpoint.id === 'pty-failed')?.acceptedSourceEndSu
).toBe(0)
const unrelated = harness.intake.acceptData(
event({
id: 'pty-unrelated',
providerGeneration: 2,
ptyIncarnation: 'incarnation-unrelated'
})
)
harness.completions[2]!.resolve()
await expect(unrelated).resolves.toMatchObject({ providerGeneration: 2 })
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
harness.intake.closeGeneration(1, 'connection_lost')
harness.intake.closeGeneration(2, 'connection_lost')
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
sourceUnits: 0,
bytes: 0,
migratingPtys: 0
})
} finally {
vi.useRealTimers()
}
})
})
@@ -0,0 +1,134 @@
import type {
SshPtyAcceptedSourceCheckpoint,
SshPtyOutputSourceObligations
} from './ssh-pty-output-source-obligations'
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
export const SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS = 10_000
export type SshPtyOutputMigrationResult =
| Readonly<{ status: 'settled'; checkpoint: SshPtyAcceptedSourceCheckpoint }>
| Readonly<{ status: 'checkpoint-unavailable'; reason: 'completion-failed' | 'timeout' }>
export type SshPtyOutputGenerationMigration = Readonly<{
byPty: ReadonlyMap<string, Promise<SshPtyOutputMigrationResult>>
completion: Promise<void>
}>
export type SshPtyTrackedModelAdmission = {
readonly key: SshPtyModelAdmissionKey
started: boolean
completion?: Promise<unknown>
}
export class SshPtyOutputModelMigration {
private readonly pendingByPty = new Map<string, Set<SshPtyTrackedModelAdmission>>()
constructor(
private readonly admission: SshPtyModelAdmission,
private readonly sourceObligations: SshPtyOutputSourceObligations,
private readonly resetModel: (providerGeneration: number, ptyId: string) => void
) {}
track(record: SshPtyTrackedModelAdmission): void {
const id = migrationKey(record.key)
const records = this.pendingByPty.get(id) ?? new Set<SshPtyTrackedModelAdmission>()
records.add(record)
this.pendingByPty.set(id, records)
record.completion?.then(
() => this.remove(id, record),
() => this.remove(id, record)
)
}
beginGeneration(
providerGeneration: number,
timeoutMs = SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS
): SshPtyOutputGenerationMigration {
const checkpoints = this.sourceObligations.acceptedCheckpoints(providerGeneration)
const keys = checkpoints.map((checkpoint) => ({
ptyId: checkpoint.id,
providerGeneration
}))
for (const key of keys) {
this.admission.beginMigration(key)
}
const byPty = new Map<string, Promise<SshPtyOutputMigrationResult>>()
for (const key of keys) {
byPty.set(key.ptyId, this.settlePty(key, timeoutMs))
}
const completion = Promise.allSettled(byPty.values()).then(() => {})
return Object.freeze({ byPty, completion })
}
private async settlePty(
key: SshPtyModelAdmissionKey,
timeoutMs: number
): Promise<SshPtyOutputMigrationResult> {
const running = Array.from(this.pendingByPty.get(migrationKey(key)) ?? []).find(
(record) => record.started
)
if (!running?.completion) {
return this.settledCheckpoint(key)
}
let timer: ReturnType<typeof setTimeout> | undefined
const timeout = new Promise<'timeout'>((resolve) => {
timer = setTimeout(() => resolve('timeout'), normalizedTimeout(timeoutMs))
timer.unref?.()
})
try {
const outcome = await Promise.race([
running.completion.then(
() => 'settled' as const,
() => 'failed' as const
),
timeout
])
if (outcome === 'settled') {
return this.settledCheckpoint(key)
}
const reason = outcome === 'timeout' ? 'timeout' : 'completion-failed'
this.resetModel(key.providerGeneration, key.ptyId)
this.admission.cancelPty(key, `ssh_model_migration_${reason}`)
if (outcome === 'timeout') {
await running.completion.catch(() => {})
}
return Object.freeze({ status: 'checkpoint-unavailable', reason })
} finally {
if (timer) {
clearTimeout(timer)
}
}
}
private settledCheckpoint(key: SshPtyModelAdmissionKey): SshPtyOutputMigrationResult {
const checkpoint = this.sourceObligations.acceptedCheckpoint(key)
if (!checkpoint) {
this.resetModel(key.providerGeneration, key.ptyId)
return Object.freeze({
status: 'checkpoint-unavailable',
reason: 'completion-failed'
})
}
return Object.freeze({ status: 'settled', checkpoint })
}
private remove(id: string, record: SshPtyTrackedModelAdmission): void {
const records = this.pendingByPty.get(id)
records?.delete(record)
if (records?.size === 0) {
this.pendingByPty.delete(id)
}
}
}
function migrationKey(key: SshPtyModelAdmissionKey): string {
return `${key.providerGeneration}\0${key.ptyId}`
}
function normalizedTimeout(timeoutMs: number): number {
return Number.isFinite(timeoutMs) && timeoutMs >= 0
? Math.floor(timeoutMs)
: SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS
}
@@ -0,0 +1,297 @@
import {
ptySourceDeliveryKey,
samePtySourceDelivery,
type PtySourceDeliveryIdentity,
type PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import type { RemoteTerminalSourceRangeConsumerHooks } from '../runtime/remote-terminal-source-range-consumer'
import type { DesktopProjectionSpan } from './ssh-pty-legacy-projection'
import type {
SshPtyOutputDataEvent,
SshPtyOutputExitEvent,
SshPtyOutputIntakeDependencies,
SshPtySourceCancellationProof,
SshPtySourceCancellationRequest
} from './ssh-pty-output-intake-contract'
import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers'
import type { SshPtySourceAdmissionReservation } from './ssh-pty-source-obligation-contract'
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
export type SshPtyOutputSourceReservation = Readonly<{
admission: SshPtySourceAdmissionReservation
span: PtySourceSpan
}>
export type SshPtySourceCancellationProofCommit = Readonly<{
identity: PtySourceDeliveryIdentity
proof: SshPtySourceCancellationProof
}>
export type SshPtyAcceptedSourceCheckpoint = Readonly<{
id: string
providerGeneration: number
clientGeneration: number
ownerGeneration: number
ptyIncarnation: string
deliveryToken: string
acceptedSourceEndSu: number
}>
export class SshPtyOutputSourceObligations {
private readonly coordinator: SshPtySourceObligationCoordinator
private readonly remoteConsumers: SshPtyRemoteSourceRangeConsumers
private readonly openedTokens = new Set<string>()
private readonly identityByPty = new Map<string, PtySourceDeliveryIdentity>()
constructor(publish: SshPtyOutputDataEventPublisher | undefined) {
this.coordinator = new SshPtySourceObligationCoordinator({
onTokenClosed: (identity) => this.removeIdentity(identity),
publish:
publish ??
((_providerGeneration, _batch, onSettled) =>
onSettled({ ok: false, error: new Error('SSH PTY source ACK publisher unavailable') }))
})
this.remoteConsumers = new SshPtyRemoteSourceRangeConsumers(this.coordinator)
}
get remoteHooks(): RemoteTerminalSourceRangeConsumerHooks {
return this.remoteConsumers.hooks
}
reserve(
event: SshPtyOutputDataEvent,
projection: DesktopProjectionSpan
): SshPtyOutputSourceReservation {
const span = this.toSourceSpan(event, projection)
const identity = this.sourceIdentity(span)
const tokenKey = ptySourceDeliveryKey(identity)
if (!this.openedTokens.has(tokenKey)) {
this.coordinator.open(identity, span.sourceStartSu)
this.openedTokens.add(tokenKey)
this.identityByPty.set(this.ptyKey(event), identity)
}
return Object.freeze({
span,
admission: this.coordinator.reserve(identity, span, [
'model',
'desktop',
...this.remoteConsumers.requiredConsumers(event.id)
])
})
}
commit(
reservation: SshPtyOutputSourceReservation,
ptyId: string,
modelSequenceEnd: number
): void {
this.coordinator.commit(reservation.admission)
this.remoteConsumers.trackSpan(
ptyId,
reservation.span.spanId,
reservation.admission.requiredConsumers,
modelSequenceEnd
)
}
rollback(reservation: SshPtyOutputSourceReservation): boolean {
return (
this.coordinator.rollback(reservation.admission) ||
this.coordinator.rollbackCommitted(reservation.admission)
)
}
settleModel(span: PtySourceSpan): void {
this.coordinator.settle({
identity: span,
spanId: span.spanId,
consumer: 'model',
reason: 'model-accepted'
})
}
settleDesktop(span: DesktopProjectionSpan, reason: string): void {
this.coordinator.settle({
identity: span,
spanId: span.spanId,
consumer: 'desktop',
reason
})
}
transferDesktop(span: DesktopProjectionSpan, reason: string): void {
const transition = {
identity: span,
spanId: span.spanId,
consumer: 'desktop' as const,
reason
}
if (this.coordinator.beginTransfer(transition, 'model')) {
this.coordinator.commitTransfer(transition)
}
}
sealPty(event: SshPtyOutputExitEvent): void {
const identity = this.identityByPty.get(this.ptyKey(event))
if (identity) {
this.coordinator.seal(identity)
}
}
markExitPublished(event: SshPtyOutputExitEvent): void {
const identity = this.identityByPty.get(this.ptyKey(event))
if (identity) {
this.coordinator.markExitPublished(identity)
}
}
whenPtyTerminal(event: SshPtyOutputExitEvent): Promise<void> {
const identity = this.identityByPty.get(this.ptyKey(event))
return identity ? this.coordinator.whenTerminal(identity) : Promise.resolve()
}
async requestPtyCancellationProof(
event: SshPtyOutputExitEvent,
cancel: (request: SshPtySourceCancellationRequest) => Promise<SshPtySourceCancellationProof>
): Promise<SshPtySourceCancellationProofCommit | null> {
const identity = this.identityByPty.get(this.ptyKey(event))
if (!identity) {
return null
}
const request = this.coordinator.beginExitTimeout(identity)
const proof = await cancel(request)
return Object.freeze({ identity, proof })
}
commitPtyCancellationProof(commit: SshPtySourceCancellationProofCommit): void {
this.coordinator.applyCancellationProof(commit.identity, commit.proof)
}
applyCancellationProof(
event: SshPtyOutputExitEvent,
proof: SshPtySourceCancellationProof
): boolean {
const identity = this.identityByPty.get(this.ptyKey(event))
if (!identity) {
return false
}
this.coordinator.applyCancellationProof(identity, proof)
return true
}
applyRecoveryCancellationProof(
event: SshPtyOutputExitEvent,
proof: SshPtySourceCancellationProof
): void {
const identity = this.identityByPty.get(this.ptyKey(event))
if (identity) {
this.coordinator.applyRecoveryCancellationProof(identity, proof)
}
}
closeGeneration(providerGeneration: number, reason: string): void {
this.remoteConsumers.closeGeneration(providerGeneration, reason)
this.coordinator.closeGeneration(providerGeneration, reason)
const prefix = `${providerGeneration}\0`
for (const key of this.openedTokens) {
if (key.startsWith(prefix)) {
this.openedTokens.delete(key)
}
}
for (const [key, identity] of this.identityByPty) {
if (identity.providerGeneration === providerGeneration) {
this.identityByPty.delete(key)
}
}
}
acceptedCheckpoints(providerGeneration: number): readonly SshPtyAcceptedSourceCheckpoint[] {
const checkpoints: SshPtyAcceptedSourceCheckpoint[] = []
for (const identity of this.identityByPty.values()) {
if (identity.providerGeneration !== providerGeneration) {
continue
}
checkpoints.push(
Object.freeze({
id: identity.id,
providerGeneration: identity.providerGeneration,
clientGeneration: identity.clientGeneration,
ownerGeneration: identity.ownerGeneration,
ptyIncarnation: identity.ptyIncarnation,
deliveryToken: identity.deliveryToken,
acceptedSourceEndSu: this.coordinator.modelAcceptedEnd(identity)
})
)
}
return Object.freeze(checkpoints)
}
acceptedCheckpoint(key: {
ptyId: string
providerGeneration: number
}): SshPtyAcceptedSourceCheckpoint | null {
return (
this.acceptedCheckpoints(key.providerGeneration).find(
(checkpoint) => checkpoint.id === key.ptyId
) ?? null
)
}
dispose(): void {
this.coordinator.dispose()
}
getDebugSnapshot(): Readonly<{ openedTokens: number; ptyIdentities: number }> {
return Object.freeze({
openedTokens: this.openedTokens.size,
ptyIdentities: this.identityByPty.size
})
}
private toSourceSpan(
event: SshPtyOutputDataEvent,
projection: DesktopProjectionSpan
): PtySourceSpan {
return Object.freeze({
id: projection.id,
providerGeneration: event.providerGeneration,
clientGeneration: projection.clientGeneration,
ownerGeneration: projection.ownerGeneration,
ptyIncarnation: event.ptyIncarnation,
deliveryToken: projection.deliveryToken,
spanId: projection.spanId,
sourceStartSu: projection.sourceStartSu,
sourceEndSu: projection.sourceEndSu,
displayStart: projection.displayStart,
displayEnd: projection.displayEnd,
splittable: projection.splittable,
transform: projection.transform,
data: event.data
})
}
private sourceIdentity(source: PtySourceSpan): PtySourceDeliveryIdentity {
return source
}
private ptyKey(event: {
id: string
providerGeneration: number
ptyIncarnation: string
}): string {
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
}
private removeIdentity(identity: PtySourceDeliveryIdentity): void {
this.openedTokens.delete(ptySourceDeliveryKey(identity))
for (const [key, candidate] of this.identityByPty) {
if (samePtySourceDelivery(candidate, identity)) {
this.identityByPty.delete(key)
}
}
}
}
type SshPtyOutputDataEventPublisher = NonNullable<
SshPtyOutputIntakeDependencies['publishSourceAck']
>
@@ -0,0 +1,112 @@
import type { ProjectionRecord } from './ssh-pty-legacy-projection-record'
type ProjectionTerminalWaiter = {
providerGeneration: number
ptyIncarnation: string
resolve: () => void
}
function matchesProjection(
record: ProjectionRecord | undefined,
providerGeneration: number,
ptyIncarnation: string
): boolean {
const identity = record?.semantics.identity
return (
identity?.providerGeneration === providerGeneration &&
identity.ptyIncarnation === ptyIncarnation
)
}
export function unpublishedProjectionIds(
records: ReadonlyMap<string, ProjectionRecord>,
ids: readonly string[],
providerGeneration: number,
ptyIncarnation: string
): string[] {
return ids.filter((id) => {
const record = records.get(id)
return (
record?.state === 'committed' && matchesProjection(record, providerGeneration, ptyIncarnation)
)
})
}
export function projectionHasOpen(
records: ReadonlyMap<string, ProjectionRecord>,
idsByPty: ReadonlyMap<string, readonly string[]>,
ptyId: string
): (providerGeneration: number, ptyIncarnation: string) => boolean {
return (providerGeneration, ptyIncarnation) =>
(idsByPty.get(ptyId) ?? []).some((id) =>
matchesProjection(records.get(id), providerGeneration, ptyIncarnation)
)
}
export function resolveProjectionTerminality(
terminality: SshPtyProjectionTerminality,
records: ReadonlyMap<string, ProjectionRecord>,
idsByPty: ReadonlyMap<string, readonly string[]>,
ptyId: string
): void {
terminality.resolve(ptyId, projectionHasOpen(records, idsByPty, ptyId))
}
export class SshPtyProjectionTerminality {
private readonly waitersByPty = new Map<string, ProjectionTerminalWaiter[]>()
whenTerminal(
ptyId: string,
providerGeneration: number,
ptyIncarnation: string,
hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean
): Promise<void> {
if (!hasOpen(providerGeneration, ptyIncarnation)) {
return Promise.resolve()
}
return new Promise((resolve) => {
const waiters = this.waitersByPty.get(ptyId) ?? []
waiters.push({ providerGeneration, ptyIncarnation, resolve })
this.waitersByPty.set(ptyId, waiters)
})
}
resolve(
ptyId: string,
hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean
): void {
const waiters = this.waitersByPty.get(ptyId)
if (!waiters) {
return
}
const pending = waiters.filter((waiter) =>
hasOpen(waiter.providerGeneration, waiter.ptyIncarnation)
)
for (const waiter of waiters) {
if (!pending.includes(waiter)) {
waiter.resolve()
}
}
if (pending.length > 0) {
this.waitersByPty.set(ptyId, pending)
} else {
this.waitersByPty.delete(ptyId)
}
}
closeGeneration(providerGeneration: number): void {
for (const [ptyId, waiters] of this.waitersByPty) {
const pending = waiters.filter((waiter) => waiter.providerGeneration !== providerGeneration)
for (const waiter of waiters) {
if (waiter.providerGeneration === providerGeneration) {
waiter.resolve()
}
}
if (pending.length > 0) {
this.waitersByPty.set(ptyId, pending)
} else {
this.waitersByPty.delete(ptyId)
}
}
}
}
@@ -0,0 +1,451 @@
import { describe, expect, it, vi } from 'vitest'
import type {
PtySourceDeliveryIdentity,
PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers'
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
const identity: PtySourceDeliveryIdentity = {
id: 'pty-1',
providerGeneration: 1,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1'
}
function span(spanId: string, overrides: Partial<PtySourceSpan> = {}): PtySourceSpan {
return {
...identity,
spanId,
sourceStartSu: 0,
sourceEndSu: 4,
displayStart: 0,
displayEnd: 4,
data: 'data',
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
...overrides
}
}
function range(
spanId: string,
overrides: Partial<TerminalOutputSourceRange> = {}
): TerminalOutputSourceRange {
return {
id: 'pty-1',
spanId,
providerGeneration: 1,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4,
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
...overrides
}
}
function createCoordinator(): SshPtySourceObligationCoordinator {
return new SshPtySourceObligationCoordinator({
publish: vi.fn(),
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
}
describe('SshPtyRemoteSourceRangeConsumers', () => {
it('snapshots membership and settles only the current stream generation', () => {
const ledger = createCoordinator()
const progress = vi.fn()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger, progress)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
expect(consumers.hooks.attach(stream)).toBe(true)
const sourceSpan = span('span-1')
const reservation = ledger.reserve(identity, sourceSpan, [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', sourceSpan.spanId, reservation.requiredConsumers, 4)
consumers.hooks.settle({ ...stream, streamGeneration: 'stale' }, [range('span-1')])
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
consumers.hooks.settle(stream, [range('span-1')])
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled')
expect(progress).toHaveBeenCalledTimes(1)
})
it.each(['headless', 'renderer'] as const)(
'commits remaining mappings only after a %s snapshot publication',
(source) => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
expect(replacement).not.toBeNull()
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
state: 'transferring',
to: 'remote:snapshot:consumer-1'
})
expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 3 })).toBe(false)
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('transferring')
expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 4 })).toBe(true)
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
state: 'transferred',
to: 'remote:snapshot:consumer-1'
})
expect(consumers.requiredConsumers('pty-1')).toEqual(['remote:consumer-1'])
}
)
it('reserves only spans covered by the authoritative snapshot sequence', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const covered = ledger.reserve(identity, span('span-covered'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(covered)
consumers.trackSpan('pty-1', 'span-covered', covered.requiredConsumers, 4)
const trailingSpan = {
...span('span-trailing'),
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
}
const trailing = ledger.reserve(identity, trailingSpan, [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(trailing)
consumers.trackSpan('pty-1', 'span-trailing', trailing.requiredConsumers, 8)
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
expect(replacement).not.toBeNull()
expect(ledger.obligation('span-covered', 'remote:consumer-1')).toMatchObject({
state: 'transferring'
})
expect(ledger.obligation('span-trailing', 'remote:consumer-1')).toMatchObject({
state: 'open'
})
})
it('rolls a failed replacement publication back to the live stream obligation', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
expect(consumers.hooks.rollbackReplacement(replacement!, 'snapshot-write-failed')).toBe(true)
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
state: 'open'
})
})
it('rolls back every transfer when replacement reservation fails partway', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const first = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(first)
consumers.trackSpan('pty-1', 'span-1', first.requiredConsumers, 4)
const secondSpan = span('span-2', {
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
})
const second = ledger.reserve(identity, secondSpan, [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(second)
consumers.trackSpan('pty-1', 'span-2', second.requiredConsumers, 8)
const beginTransfer = ledger.beginTransfer.bind(ledger)
vi.spyOn(ledger, 'beginTransfer').mockImplementation((transition, replacement) => {
if (transition.spanId === 'span-2') {
throw new Error('injected partial reserve failure')
}
return beginTransfer(transition, replacement)
})
expect(() => consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')).toThrowError(
'injected partial reserve failure'
)
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open')
})
it('rejects a replacement whose exact transfer state changed before commit', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const sourceSpan = span('span-1')
const admission = ledger.reserve(identity, sourceSpan, [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(admission)
consumers.trackSpan('pty-1', 'span-1', admission.requiredConsumers, 4)
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')!
const transition = {
identity: sourceSpan,
spanId: sourceSpan.spanId,
consumer: 'remote:consumer-1' as const,
reason: 'concurrent-replacement'
}
expect(ledger.rollbackTransfer(transition)).toBe(true)
expect(ledger.beginTransfer(transition, 'remote:snapshot:consumer-1')).toBe(true)
expect(consumers.hooks.commitReplacement(replacement, { source: 'headless', seq: 4 })).toBe(
false
)
expect(consumers.hooks.rollbackReplacement(replacement, 'commit-rejected')).toBe(false)
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
state: 'transferring',
reason: 'concurrent-replacement'
})
})
it('settles a split source span only after its complete ordered range is acknowledged', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
consumers.hooks.settle(stream, [
range('span-1', {
sourceEndSu: 2,
displayEnd: 2,
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
})
])
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
consumers.hooks.settle(stream, [
range('span-1', {
sourceStartSu: 2,
sourceEndSu: 4,
displayStart: 2,
displayEnd: 4,
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
})
])
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled')
})
it('cancels an admitted span on detach without minting a snapshot owner', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
consumers.hooks.cancel(stream, [], 'stream-detached')
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
state: 'canceled',
reason: 'stream-detached'
})
expect(consumers.requiredConsumers('pty-1')).toEqual([])
})
it('rolls back a pending replacement before disconnect cancellation', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
consumers.hooks.cancel(stream, [], 'connection-closed')
expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe(
false
)
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
state: 'canceled',
reason: 'connection-closed'
})
})
it('rejects stale stream generations without changing the current obligation', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
expect(() =>
consumers.hooks.reserveReplacement(
{ ...stream, streamGeneration: 'stale' },
4,
'initial-snapshot'
)
).toThrow('stale')
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
})
it('rolls back replacement admission before provider-generation close', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
consumers.closeGeneration(identity.providerGeneration, 'provider-replaced')
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe(
false
)
})
it('detaches cleanly after cancellation proof reclaims tracked spans', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
ledger.seal(identity)
ledger.beginExitTimeout(identity)
ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 })
expect(() => consumers.hooks.settle(stream, [range('span-1')])).not.toThrow()
expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow()
expect(consumers.requiredConsumers('pty-1')).toEqual([])
})
it('rejects and rolls back a replacement after proof reclaims its covered spans', () => {
const ledger = createCoordinator()
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
ledger.open(identity)
consumers.hooks.attach(stream)
const reservation = ledger.reserve(identity, span('span-1'), [
'model',
...consumers.requiredConsumers('pty-1')
])
ledger.commit(reservation)
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
const recoveredIdentity = {
...identity,
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-2'
}
ledger.open(recoveredIdentity, 4)
const recovered = ledger.reserve(
recoveredIdentity,
span('span-2', {
...recoveredIdentity,
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
}),
['model', ...consumers.requiredConsumers('pty-1')]
)
ledger.commit(recovered)
consumers.trackSpan('pty-1', 'span-2', recovered.requiredConsumers, 8)
const replacement = consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')
ledger.seal(identity)
ledger.beginExitTimeout(identity)
ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 })
const queuedOutput = ['covered-output', 'surviving-output']
const committed = consumers.hooks.commitReplacement(replacement!, {
source: 'headless',
seq: 8
})
if (committed) {
queuedOutput.splice(0)
}
expect(committed).toBe(false)
expect(queuedOutput).toEqual(['covered-output', 'surviving-output'])
expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(true)
expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open')
expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(false)
expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow()
expect(consumers.requiredConsumers('pty-1')).toEqual([])
})
})
@@ -0,0 +1,285 @@
import {
sameTerminalOutputSourceIdentity,
type TerminalOutputSourceRange
} from '../../shared/terminal-output-source-range'
import type {
RemoteTerminalSourceRangeConsumerHooks,
RemoteTerminalSourceRangeStreamIdentity
} from '../runtime/remote-terminal-source-range-consumer'
import type { PtySourceSpan } from '../../shared/pty-source-credit-contract'
import type { SshPtySourceConsumerId } from './ssh-pty-source-obligation-contract'
import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
import { SshPtyRemoteSourceRangeReplacements } from './ssh-pty-remote-source-range-replacement'
function remoteConsumerId(
identity: RemoteTerminalSourceRangeStreamIdentity
): SshPtySourceConsumerId {
return `remote:${identity.consumerId}`
}
function uniqueSpanIds(ranges: readonly TerminalOutputSourceRange[]): string[] {
return Array.from(new Set(ranges.map((range) => range.spanId)))
}
type RemoteConsumerState = {
streamGeneration: string
spans: Map<
string,
Readonly<{
identity: PtySourceSpan
modelSequenceEnd: number
}>
>
ackedEndBySpan: Map<string, number>
}
export class SshPtyRemoteSourceRangeConsumers {
private readonly consumersByPty = new Map<string, Map<string, RemoteConsumerState>>()
private readonly replacements: SshPtyRemoteSourceRangeReplacements
constructor(
private readonly coordinator: SshPtySourceObligationCoordinator,
private readonly onProgress: (range: TerminalOutputSourceRange) => void = () => {}
) {
this.replacements = new SshPtyRemoteSourceRangeReplacements(coordinator)
}
readonly hooks: RemoteTerminalSourceRangeConsumerHooks = {
attach: (identity) => this.attach(identity),
settle: (identity, ranges) => this.settle(identity, ranges),
reserveReplacement: (identity, requiredSeq, reason) =>
this.reserveReplacement(identity, requiredSeq, reason),
commitReplacement: (reservation, publication) =>
this.commitReplacement(reservation, publication),
rollbackReplacement: (reservation, reason) => this.rollbackReplacement(reservation, reason),
cancel: (identity, ranges, reason) => this.cancel(identity, ranges, reason)
}
requiredConsumers(ptyId: string): readonly SshPtySourceConsumerId[] {
return Object.freeze(
Array.from(this.consumersByPty.get(ptyId)?.keys() ?? []).map(
(consumerId) => `remote:${consumerId}` as const
)
)
}
trackSpan(
ptyId: string,
spanId: string,
requiredConsumers: readonly SshPtySourceConsumerId[],
modelSequenceEnd: number
): void {
if (!Number.isSafeInteger(modelSequenceEnd) || modelSequenceEnd < 0) {
throw new Error('ssh_remote_source_range_model_sequence_invalid')
}
for (const [consumerId, state] of this.consumersByPty.get(ptyId) ?? []) {
if (requiredConsumers.includes(`remote:${consumerId}`)) {
state.spans.set(
spanId,
Object.freeze({
identity: this.coordinator.spanIdentity(spanId),
modelSequenceEnd
})
)
}
}
}
closeGeneration(providerGeneration: number, reason: string): void {
this.replacements.closeGeneration(providerGeneration, reason)
for (const consumers of this.consumersByPty.values()) {
for (const state of consumers.values()) {
for (const [spanId, tracked] of state.spans) {
if (tracked.identity.providerGeneration === providerGeneration) {
state.spans.delete(spanId)
state.ackedEndBySpan.delete(spanId)
}
}
}
}
}
private attach(identity: RemoteTerminalSourceRangeStreamIdentity): boolean {
const consumers =
this.consumersByPty.get(identity.ptyId) ?? new Map<string, RemoteConsumerState>()
const current = consumers.get(identity.consumerId)
if (current && current.streamGeneration !== identity.streamGeneration) {
return false
}
consumers.set(
identity.consumerId,
current ?? {
streamGeneration: identity.streamGeneration,
spans: new Map(),
ackedEndBySpan: new Map()
}
)
this.consumersByPty.set(identity.ptyId, consumers)
return true
}
private settle(
identity: RemoteTerminalSourceRangeStreamIdentity,
ranges: readonly TerminalOutputSourceRange[]
): void {
if (!this.isCurrent(identity)) {
return
}
const state = this.requireState(identity)
const consumer = remoteConsumerId(identity)
const nextEnds = new Map(state.ackedEndBySpan)
const completed = new Set<string>()
for (const range of ranges) {
const tracked = state.spans.get(range.spanId)
if (!tracked) {
continue
}
const source = tracked.identity
if (!this.coordinator.hasRetainedSpan(range.spanId)) {
state.spans.delete(range.spanId)
nextEnds.delete(range.spanId)
continue
}
const currentEnd = nextEnds.get(range.spanId) ?? source.sourceStartSu
if (
!sameTerminalOutputSourceIdentity(source, range) ||
range.sourceStartSu !== currentEnd ||
range.sourceEndSu > source.sourceEndSu
) {
throw new Error('ssh_remote_source_range_settlement_invalid')
}
nextEnds.set(range.spanId, range.sourceEndSu)
if (range.sourceEndSu === source.sourceEndSu) {
completed.add(range.spanId)
}
}
state.ackedEndBySpan = nextEnds
for (const spanId of completed) {
const tracked = state.spans.get(spanId)
if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) {
state.spans.delete(spanId)
state.ackedEndBySpan.delete(spanId)
continue
}
const source = tracked.identity
this.coordinator.settle({
identity: source,
spanId,
consumer,
reason: 'remote-frame-ack'
})
state.spans.delete(spanId)
state.ackedEndBySpan.delete(spanId)
}
for (const range of ranges) {
this.onProgress(range)
}
}
private reserveReplacement(
identity: RemoteTerminalSourceRangeStreamIdentity,
requiredSeq: number,
reason: string
): ReturnType<SshPtyRemoteSourceRangeReplacements['reserve']> {
if (!this.isCurrent(identity)) {
throw new Error('ssh_remote_source_range_stale_generation')
}
const state = this.requireState(identity)
for (const spanId of state.spans.keys()) {
if (!this.coordinator.hasRetainedSpan(spanId)) {
state.spans.delete(spanId)
state.ackedEndBySpan.delete(spanId)
}
}
const spanIds = Array.from(state.spans)
.filter(([, tracked]) => tracked.modelSequenceEnd <= requiredSeq)
.map(([spanId]) => spanId)
return this.replacements.reserve(identity, spanIds, requiredSeq, reason)
}
private commitReplacement(
reservation: Parameters<SshPtyRemoteSourceRangeReplacements['commit']>[0],
publication: Parameters<SshPtyRemoteSourceRangeReplacements['commit']>[1]
): boolean {
return this.replacements.commit(
reservation,
publication,
this.isCurrent(reservation.identity),
(spanIds) => {
const state = this.requireState(reservation.identity)
for (const spanId of spanIds) {
state.spans.delete(spanId)
state.ackedEndBySpan.delete(spanId)
}
}
)
}
private rollbackReplacement(
reservation: Parameters<SshPtyRemoteSourceRangeReplacements['rollback']>[0],
reason: string
): boolean {
return this.replacements.rollback(reservation, reason)
}
private cancel(
identity: RemoteTerminalSourceRangeStreamIdentity,
ranges: readonly TerminalOutputSourceRange[],
reason: string
): void {
if (!this.isCurrent(identity)) {
return
}
this.replacements.rollbackIdentity(identity, `${reason}-replacement-aborted`)
const consumer = remoteConsumerId(identity)
const state = this.requireState(identity)
const spanIds = new Set([
...state.spans.keys(),
...uniqueSpanIds(ranges).filter((spanId) => state.spans.has(spanId))
])
for (const spanId of spanIds) {
const tracked = state.spans.get(spanId)
if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) {
continue
}
const source = tracked.identity
const transition = { identity: source, spanId, consumer, reason }
if (this.coordinator.beginTransfer(transition, consumer)) {
this.coordinator.cancelTransfer(transition)
}
}
this.detachIdentity(identity)
for (const range of ranges) {
this.onProgress(range)
}
}
private isCurrent(identity: RemoteTerminalSourceRangeStreamIdentity): boolean {
return (
this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)?.streamGeneration ===
identity.streamGeneration
)
}
private requireState(identity: RemoteTerminalSourceRangeStreamIdentity): RemoteConsumerState {
const state = this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)
if (!state || state.streamGeneration !== identity.streamGeneration) {
throw new Error('ssh_remote_source_range_stale_generation')
}
return state
}
private detachIdentity(identity: RemoteTerminalSourceRangeStreamIdentity): void {
const consumers = this.consumersByPty.get(identity.ptyId)
if (
!consumers ||
consumers.get(identity.consumerId)?.streamGeneration !== identity.streamGeneration
) {
return
}
consumers.delete(identity.consumerId)
if (consumers.size === 0) {
this.consumersByPty.delete(identity.ptyId)
}
}
}
@@ -0,0 +1,194 @@
import type {
RemoteTerminalSourceRangeReplacementPublication,
RemoteTerminalSourceRangeReplacementReservation,
RemoteTerminalSourceRangeStreamIdentity
} from '../runtime/remote-terminal-source-range-consumer'
import type { PtySourceSpan } from '../../shared/pty-source-credit-contract'
import type {
SshPtySourceConsumerId,
SshPtySourceObligationState
} from './ssh-pty-source-obligation-contract'
import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
type ReplacementSpanRecord = Readonly<{
source: PtySourceSpan
transferState: SshPtySourceObligationState
}>
type ReplacementReservationRecord = {
reservation: RemoteTerminalSourceRangeReplacementReservation
spans: readonly ReplacementSpanRecord[]
consumer: SshPtySourceConsumerId
replacement: SshPtySourceConsumerId
reason: string
}
function remoteConsumerId(
identity: RemoteTerminalSourceRangeStreamIdentity
): SshPtySourceConsumerId {
return `remote:${identity.consumerId}`
}
export class SshPtyRemoteSourceRangeReplacements {
private readonly reservations = new Map<string, ReplacementReservationRecord>()
private nextReservationId = 1
constructor(private readonly coordinator: SshPtySourceObligationCoordinator) {}
reserve(
identity: RemoteTerminalSourceRangeStreamIdentity,
spanIds: readonly string[],
requiredSeq: number,
reason: string
): RemoteTerminalSourceRangeReplacementReservation | null {
if (spanIds.length === 0) {
return null
}
if (!Number.isSafeInteger(requiredSeq) || requiredSeq < 0) {
throw new Error('ssh_remote_source_range_replacement_sequence_invalid')
}
const consumer = remoteConsumerId(identity)
const replacement = `remote:snapshot:${identity.consumerId}` as const
const spans = spanIds.map((spanId) => this.coordinator.spanIdentity(spanId))
for (const { spanId } of spans) {
if (this.coordinator.obligation(spanId, consumer).state !== 'open') {
throw new Error('ssh_remote_source_range_transfer_invalid')
}
}
const transferred: ReplacementSpanRecord[] = []
try {
for (const source of spans) {
const { spanId } = source
const transition = { identity: source, spanId, consumer, reason }
if (!this.coordinator.beginTransfer(transition, replacement)) {
throw new Error('ssh_remote_source_range_transfer_invalid')
}
const transferState = this.coordinator.obligation(spanId, consumer)
if (transferState.state !== 'transferring' || transferState.to !== replacement) {
this.coordinator.rollbackTransfer(transition)
throw new Error('ssh_remote_source_range_transfer_invalid')
}
transferred.push(Object.freeze({ source, transferState }))
}
} catch (error) {
for (const span of transferred.toReversed()) {
this.rollbackExactSpan(span, consumer, reason)
}
throw error
}
const reservation = Object.freeze({
reservationId: `remote-source-replacement:${this.nextReservationId++}`,
identity: Object.freeze({ ...identity }),
requiredSeq
})
this.reservations.set(reservation.reservationId, {
reservation,
spans: Object.freeze(transferred),
consumer,
replacement,
reason
})
return reservation
}
commit(
reservation: RemoteTerminalSourceRangeReplacementReservation,
publication: RemoteTerminalSourceRangeReplacementPublication,
isCurrent: boolean,
onCommitted: (spanIds: readonly string[]) => void
): boolean {
const record = this.reservations.get(reservation.reservationId)
if (
!record ||
record.reservation !== reservation ||
!isCurrent ||
!Number.isSafeInteger(publication.seq) ||
publication.seq < reservation.requiredSeq ||
(publication.source !== 'headless' && publication.source !== 'renderer')
) {
return false
}
if (
record.spans.some(({ source, transferState }) => {
const { spanId } = source
if (!this.coordinator.hasRetainedSpan(spanId)) {
return true
}
return this.coordinator.obligation(spanId, record.consumer) !== transferState
})
) {
return false
}
for (const { source } of record.spans) {
const { spanId } = source
if (!this.coordinator.hasRetainedSpan(spanId)) {
continue
}
if (
!this.coordinator.commitTransfer({ identity: source, spanId, consumer: record.consumer })
) {
throw new Error('ssh_remote_source_range_replacement_commit_invalid')
}
}
this.reservations.delete(reservation.reservationId)
onCommitted(record.spans.map(({ source }) => source.spanId))
return true
}
rollback(reservation: RemoteTerminalSourceRangeReplacementReservation, reason: string): boolean {
const record = this.reservations.get(reservation.reservationId)
if (!record || record.reservation !== reservation) {
return false
}
this.reservations.delete(reservation.reservationId)
let rolledBack = true
for (const span of record.spans) {
const { spanId } = span.source
if (!this.coordinator.hasRetainedSpan(spanId)) {
continue
}
rolledBack = this.rollbackExactSpan(span, record.consumer, reason) && rolledBack
}
return rolledBack
}
rollbackIdentity(identity: RemoteTerminalSourceRangeStreamIdentity, reason: string): void {
for (const record of Array.from(this.reservations.values())) {
if (
record.reservation.identity.ptyId === identity.ptyId &&
record.reservation.identity.consumerId === identity.consumerId &&
record.reservation.identity.streamGeneration === identity.streamGeneration
) {
this.rollback(record.reservation, reason)
}
}
}
closeGeneration(providerGeneration: number, reason: string): void {
for (const record of Array.from(this.reservations.values())) {
if (record.spans.some(({ source }) => source.providerGeneration === providerGeneration)) {
this.rollback(record.reservation, `${reason}-replacement-aborted`)
}
}
}
private rollbackExactSpan(
span: ReplacementSpanRecord,
consumer: SshPtySourceConsumerId,
reason: string
): boolean {
const { source, transferState } = span
if (
!this.coordinator.hasRetainedSpan(source.spanId) ||
this.coordinator.obligation(source.spanId, consumer) !== transferState
) {
return false
}
return this.coordinator.rollbackTransfer({
identity: source,
spanId: source.spanId,
consumer,
reason
})
}
}
@@ -0,0 +1,168 @@
import { describe, expect, it, vi } from 'vitest'
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
import { SshPtySourceAckCoalescer } from './ssh-pty-source-ack-coalescer'
function publication(token: number, endSu: number, settled = vi.fn(), providerGeneration = 1) {
const identity = {
id: `pty-${token}`,
providerGeneration,
clientGeneration: 1,
ownerGeneration: 1,
ptyIncarnation: `incarnation-${token}`,
deliveryToken: `token-${token}`
}
return {
identity,
ack: {
id: identity.id,
clientGeneration: 1,
ownerGeneration: 1,
deliveryToken: identity.deliveryToken,
creditedEndSu: endSu
},
onSettled: settled
}
}
describe('SshPtySourceAckCoalescer', () => {
it('coalesces cumulative values and advances them only from the write callback', () => {
const writes: {
batch: PtySourceCreditAckBatch
settle: (result: { ok: true } | { ok: false; error: Error }) => void
}[] = []
const firstSettled = vi.fn()
const latestSettled = vi.fn()
const coalescer = new SshPtySourceAckCoalescer({
publish: (_providerGeneration, batch, settle) => writes.push({ batch, settle }),
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coalescer.enqueue(publication(1, 10, firstSettled))
coalescer.enqueue(publication(1, 20, latestSettled))
coalescer.flush()
expect(writes[0].batch.acknowledgements).toEqual([
expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 20 })
])
expect(latestSettled).not.toHaveBeenCalled()
writes[0].settle({ ok: true })
expect(latestSettled).toHaveBeenCalledWith({ ok: true })
expect(firstSettled).toHaveBeenCalledWith({ ok: true })
})
it('limits one batch to 64 tokens and gives the remainder another turn', () => {
const batches: PtySourceCreditAckBatch[] = []
const scheduled: (() => void)[] = []
const coalescer = new SshPtySourceAckCoalescer({
publish: (_providerGeneration, batch, settle) => {
batches.push(batch)
settle({ ok: true })
},
schedule: (callback) => {
scheduled.push(callback)
return scheduled.length as unknown as ReturnType<typeof setTimeout>
},
cancelSchedule: vi.fn()
})
for (let token = 0; token < 70; token++) {
coalescer.enqueue(publication(token, 1))
}
coalescer.flush()
expect(batches[0].acknowledgements).toHaveLength(64)
scheduled.at(-1)!()
expect(batches[1].acknowledgements).toHaveLength(6)
})
it('never mixes provider generations in one transport batch', () => {
const generations: number[] = []
const coalescer = new SshPtySourceAckCoalescer({
publish: (providerGeneration, _batch, settle) => {
generations.push(providerGeneration)
settle({ ok: true })
},
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coalescer.enqueue(publication(1, 1, vi.fn(), 1))
coalescer.enqueue(publication(2, 1, vi.fn(), 2))
coalescer.flush()
coalescer.flush()
expect(generations).toEqual([1, 2])
})
it('settles every entry as failed on a synchronous send error', () => {
const settled = vi.fn()
const coalescer = new SshPtySourceAckCoalescer({
publish: () => {
throw new Error('send failed')
},
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coalescer.enqueue(publication(1, 10, settled))
coalescer.flush()
expect(settled).toHaveBeenCalledWith({
ok: false,
error: expect.objectContaining({ message: 'send failed' })
})
})
it('promotes a pending interval flush to immediate at the source threshold', () => {
const delays: number[] = []
const cancelSchedule = vi.fn()
const coalescer = new SshPtySourceAckCoalescer({
publish: vi.fn(),
schedule: (_callback, delayMs) => {
delays.push(delayMs)
return delays.length as unknown as ReturnType<typeof setTimeout>
},
cancelSchedule
})
coalescer.enqueue(publication(1, 1))
coalescer.enqueue(publication(1, 64 * 1024))
expect(delays).toEqual([8, 0])
expect(cancelSchedule).toHaveBeenCalledOnce()
})
it('fails queued callbacks exactly once during cleanup', () => {
const settled = vi.fn()
const coalescer = new SshPtySourceAckCoalescer({
publish: vi.fn(),
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coalescer.enqueue(publication(1, 10, settled))
coalescer.dispose()
coalescer.dispose()
expect(settled).toHaveBeenCalledOnce()
})
it('owns an in-flight callback until dispose and ignores its late transport callback', () => {
let transportSettle!: (result: { ok: true } | { ok: false; error: Error }) => void
const settled = vi.fn()
const coalescer = new SshPtySourceAckCoalescer({
publish: (_providerGeneration, _batch, settle) => {
transportSettle = settle
},
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coalescer.enqueue(publication(1, 10, settled))
coalescer.flush()
coalescer.dispose('generation closed')
transportSettle({ ok: true })
expect(settled).toHaveBeenCalledOnce()
expect(settled).toHaveBeenCalledWith({
ok: false,
error: expect.objectContaining({ message: 'generation closed' })
})
})
})
@@ -0,0 +1,230 @@
import {
MAX_PTY_ACK_ENTRIES,
type PtySourceCreditAckBatch
} from '../../shared/pty-source-credit-contract'
import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract'
export const SSH_PTY_ACK_FLUSH_MS = 8
export const SSH_PTY_ACK_EAGER_ADVANCE_SU = 64 * 1024
const ACK_PUBLICATION_WATERMARK_LIMIT = 1024
type AckSettlement = { ok: true } | { ok: false; error: Error }
type CoalescedEntry = {
publication: SshPtySourceAckPublication
members: SshPtySourceAckPublication[]
}
type InFlightBatch = {
entries: CoalescedEntry[]
settled: boolean
}
export type SshPtySourceAckCoalescerOptions = {
publish: (
providerGeneration: number,
batch: PtySourceCreditAckBatch,
onSettled: (result: AckSettlement) => void
) => void
schedule?: (callback: () => void, delayMs: number) => ReturnType<typeof setTimeout>
cancelSchedule?: (timer: ReturnType<typeof setTimeout>) => void
onTokenClosed?: (identity: SshPtySourceAckPublication['identity']) => void
}
function ackKey(publication: SshPtySourceAckPublication): string {
const { identity } = publication
return `${identity.providerGeneration}\0${identity.clientGeneration}\0${identity.ownerGeneration}\0${identity.id}\0${identity.ptyIncarnation}\0${identity.deliveryToken}`
}
export class SshPtySourceAckCoalescer {
private readonly pending = new Map<string, CoalescedEntry>()
private readonly lastPublishedEndByToken = new Map<string, number>()
private readonly schedule: NonNullable<SshPtySourceAckCoalescerOptions['schedule']>
private readonly cancelSchedule: NonNullable<SshPtySourceAckCoalescerOptions['cancelSchedule']>
private timer: ReturnType<typeof setTimeout> | null = null
private timerDelayMs: number | null = null
private inFlight: InFlightBatch | null = null
private disposed = false
constructor(private readonly options: SshPtySourceAckCoalescerOptions) {
this.schedule = options.schedule ?? ((callback, delayMs) => setTimeout(callback, delayMs))
this.cancelSchedule = options.cancelSchedule ?? clearTimeout
}
enqueue(publication: SshPtySourceAckPublication): void {
if (this.disposed) {
publication.onSettled({ ok: false, error: new Error('SSH PTY ACK coalescer disposed') })
return
}
const key = ackKey(publication)
const current = this.pending.get(key)
if (!current) {
this.pending.set(key, { publication, members: [publication] })
} else {
current.members.push(publication)
if (publication.ack.creditedEndSu > current.publication.ack.creditedEndSu) {
current.publication = publication
}
}
const lastPublished = this.lastPublishedEndByToken.get(key) ?? 0
const eager = publication.ack.creditedEndSu - lastPublished >= SSH_PTY_ACK_EAGER_ADVANCE_SU
this.requestFlush(eager ? 0 : SSH_PTY_ACK_FLUSH_MS)
}
flush(): void {
if (this.disposed || this.inFlight || this.pending.size === 0) {
return
}
if (this.timer) {
this.cancelSchedule(this.timer)
this.timer = null
this.timerDelayMs = null
}
const providerGeneration = this.pending.values().next().value!.publication
.identity.providerGeneration
const selected = Array.from(this.pending.entries())
.filter(([, entry]) => entry.publication.identity.providerGeneration === providerGeneration)
.slice(0, MAX_PTY_ACK_ENTRIES)
for (const [key] of selected) {
this.pending.delete(key)
}
const batch: InFlightBatch = {
entries: selected.map(([, entry]) => entry),
settled: false
}
this.inFlight = batch
const settle = (result: AckSettlement): void => this.settleBatch(batch, result)
try {
this.options.publish(
providerGeneration,
Object.freeze({
acknowledgements: Object.freeze(batch.entries.map((entry) => entry.publication.ack))
}),
settle
)
} catch (error) {
settle({
ok: false,
error: error instanceof Error ? error : new Error(String(error))
})
}
}
dispose(reason = 'SSH PTY ACK coalescer disposed'): void {
if (this.disposed) {
return
}
this.disposed = true
if (this.timer) {
this.cancelSchedule(this.timer)
this.timer = null
this.timerDelayMs = null
}
const result = { ok: false as const, error: new Error(reason) }
for (const entry of this.pending.values()) {
this.settleMembers(entry, result)
}
this.pending.clear()
if (this.inFlight && !this.inFlight.settled) {
const batch = this.inFlight
batch.settled = true
this.inFlight = null
for (const entry of batch.entries) {
this.settleMembers(entry, result)
}
}
}
cancelGeneration(providerGeneration: number, reason: string): void {
const result = { ok: false as const, error: new Error(reason) }
for (const [key, entry] of this.pending) {
if (entry.publication.identity.providerGeneration === providerGeneration) {
this.pending.delete(key)
this.settleMembers(entry, result)
}
}
const batch = this.inFlight
if (batch && !batch.settled) {
const retained: CoalescedEntry[] = []
for (const entry of batch.entries) {
if (entry.publication.identity.providerGeneration === providerGeneration) {
this.settleMembers(entry, result)
} else {
retained.push(entry)
}
}
batch.entries = retained
if (retained.length === 0) {
batch.settled = true
this.inFlight = null
}
}
const prefix = `${providerGeneration}\0`
for (const key of this.lastPublishedEndByToken.keys()) {
if (key.startsWith(prefix)) {
this.lastPublishedEndByToken.delete(key)
}
}
if (!this.inFlight && this.pending.size === 0 && this.timer) {
this.cancelSchedule(this.timer)
this.timer = null
this.timerDelayMs = null
} else if (!this.inFlight && this.pending.size > 0) {
this.requestFlush(0)
}
}
get pendingCount(): number {
return this.pending.size
}
private settleBatch(batch: InFlightBatch, result: AckSettlement): void {
if (batch.settled) {
return
}
batch.settled = true
if (this.inFlight === batch) {
this.inFlight = null
}
for (const entry of batch.entries) {
if (result.ok) {
const key = ackKey(entry.publication)
this.lastPublishedEndByToken.delete(key)
this.lastPublishedEndByToken.set(key, entry.publication.ack.creditedEndSu)
while (this.lastPublishedEndByToken.size > ACK_PUBLICATION_WATERMARK_LIMIT) {
this.lastPublishedEndByToken.delete(this.lastPublishedEndByToken.keys().next().value!)
}
}
this.settleMembers(entry, result)
}
if (this.pending.size > 0) {
this.requestFlush(0)
}
}
private settleMembers(entry: CoalescedEntry, result: AckSettlement): void {
for (const publication of entry.members.splice(0)) {
publication.onSettled(result)
}
}
private requestFlush(delayMs: number): void {
if (this.inFlight || this.disposed) {
return
}
if (this.timer) {
if (this.timerDelayMs !== null && this.timerDelayMs <= delayMs) {
return
}
this.cancelSchedule(this.timer)
this.timer = null
}
this.timerDelayMs = delayMs
this.timer = this.schedule(() => {
this.timer = null
this.timerDelayMs = null
this.flush()
}, delayMs)
this.timer.unref?.()
}
}
@@ -0,0 +1,33 @@
import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract'
import { reclaimPublishedSourcePrefix, type TokenRecord } from './ssh-pty-source-obligation-state'
export function createSshPtySourceAckPublication(
token: TokenRecord,
endSu: number,
spanOwners: Map<string, TokenRecord>,
onPublished: () => void
): SshPtySourceAckPublication {
let settled = false
return Object.freeze({
identity: token.identity,
ack: Object.freeze({
id: token.identity.id,
clientGeneration: token.identity.clientGeneration,
ownerGeneration: token.identity.ownerGeneration,
deliveryToken: token.identity.deliveryToken,
creditedEndSu: endSu
}),
onSettled: (result) => {
if (settled) {
return
}
settled = true
if (!result.ok || token.state === 'closed' || endSu > token.ackQueuedEndSu) {
return
}
token.ackPublishedEndSu = Math.max(token.ackPublishedEndSu, endSu)
reclaimPublishedSourcePrefix(token, spanOwners)
onPublished()
}
})
}
@@ -0,0 +1,86 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
import { SshPtyOutputIntake } from './ssh-pty-output-intake'
import {
installSshPtySourceAckPublisher,
publishSshPtySourceAck
} from './ssh-pty-output-intake-registry'
afterEach(() => vi.useRealTimers())
describe('SSH PTY intake to relay ACK contract', () => {
it('publishes one cumulative relay-ID ACK only after model and desktop settlement', async () => {
vi.useFakeTimers()
const batches: PtySourceCreditAckBatch[] = []
const cleanup = installSshPtySourceAckPublisher(7, (batch, onSettled) => {
batches.push(batch)
onSettled({ ok: true })
})
let sequence = 0
const intake = new SshPtyOutputIntake({
getModelSequence: () => sequence,
acceptModel: (event) => {
sequence += event.rawLength
return { sequence, completion: Promise.resolve() }
},
project: () => {},
prepareExit: () => {},
finalizeExit: () => {},
publishSourceAck: publishSshPtySourceAck
})
try {
const receipt = await intake.acceptData({
id: 'ssh:target@@relay-pty-1',
data: 'data',
providerGeneration: 7,
ptyIncarnation: 'incarnation-1',
rawLength: 4,
transformed: false,
source: {
relayPtyId: 'relay-pty-1',
spanId: 'token-1:0:4',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4
}
})
await vi.advanceTimersByTimeAsync(8)
expect(batches).toHaveLength(0)
const projectionId = receipt.projection.identity.projectionSemanticsId
intake.publishProjectionPrefix([projectionId], 4, 4)
intake.settleProjectionPrefix('ssh:target@@relay-pty-1', 4)
await vi.advanceTimersByTimeAsync(8)
expect(batches).toEqual([
{
acknowledgements: [
{
id: 'relay-pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
creditedEndSu: 4
}
]
}
])
await intake.acceptExit({
id: 'ssh:target@@relay-pty-1',
code: 0,
providerGeneration: 7,
ptyIncarnation: 'incarnation-1'
})
expect(intake.getDebugSnapshot().source).toEqual({
openedTokens: 0,
ptyIdentities: 0
})
} finally {
intake.dispose()
cleanup()
}
})
})
@@ -0,0 +1,38 @@
import type {
PtySourceCreditAck,
PtySourceDeliveryIdentity,
PtySourceSpan
} from '../../shared/pty-source-credit-contract'
export type SshPtySourceConsumerId = 'model' | 'desktop' | `remote:${string}`
export type SshPtySourceObligationState =
| Readonly<{ state: 'open' }>
| Readonly<{ state: 'transferring'; to: SshPtySourceConsumerId; reason: string }>
| Readonly<{ state: 'settled'; reason: string }>
| Readonly<{ state: 'transferred'; to: SshPtySourceConsumerId; reason: string }>
| Readonly<{ state: 'canceled'; reason: string }>
export type SshPtySourceAdmissionReservation = Readonly<{
reservationId: string
span: PtySourceSpan
requiredConsumers: readonly SshPtySourceConsumerId[]
}>
export type SshPtySourceTokenSnapshot = PtySourceDeliveryIdentity &
Readonly<{
state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed'
receivedEndSu: number
obligationsTerminalEndSu: number
ackQueuedEndSu: number
ackPublishedEndSu: number
openSpans: number
exitPublished: boolean
generationClosed: boolean
}>
export type SshPtySourceAckPublication = Readonly<{
identity: PtySourceDeliveryIdentity
ack: PtySourceCreditAck
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
}>
@@ -0,0 +1,137 @@
import { describe, expect, it, vi } from 'vitest'
import type {
PtySourceCreditAckBatch,
PtySourceDeliveryIdentity,
PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
const identity: PtySourceDeliveryIdentity = {
id: 'pty-1',
providerGeneration: 1,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1'
}
const span: PtySourceSpan = {
...identity,
spanId: 'span-1',
sourceStartSu: 0,
sourceEndSu: 4,
displayStart: 0,
displayEnd: 4,
data: 'data',
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
}
describe('SshPtySourceObligationCoordinator', () => {
it('is the single boundary from exact consumer settlement to upstream ACK publication', () => {
let written:
| {
batch: PtySourceCreditAckBatch
settle: (result: { ok: true } | { ok: false; error: Error }) => void
}
| undefined
const coordinator = new SshPtySourceObligationCoordinator({
publish: (_providerGeneration, batch, settle) => {
written = { batch, settle }
},
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coordinator.open(identity)
const reservation = coordinator.reserve(identity, span, ['model', 'desktop'])
coordinator.commit(reservation)
coordinator.settle({
identity,
spanId: span.spanId,
consumer: 'model',
reason: 'emulator-receipt'
})
coordinator.settle({
identity,
spanId: span.spanId,
consumer: 'desktop',
reason: 'renderer-parse'
})
coordinator.flushAcknowledgements()
expect(written?.batch.acknowledgements).toEqual([
expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 4 })
])
expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(0)
written?.settle({ ok: true })
expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(4)
})
it('rejects an adapter transition carrying a stale full delivery identity', () => {
const coordinator = new SshPtySourceObligationCoordinator({
publish: vi.fn(),
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coordinator.open(identity)
coordinator.commit(coordinator.reserve(identity, span, ['model']))
expect(() =>
coordinator.settle({
identity: { ...identity, ownerGeneration: 99 },
spanId: span.spanId,
consumer: 'model',
reason: 'stale'
})
).toThrow('stale')
})
it('keeps later generations publishable after closing one generation', () => {
const publish = vi.fn()
const coordinator = new SshPtySourceObligationCoordinator({
publish,
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coordinator.open(identity)
coordinator.closeGeneration(1, 'replaced')
const nextIdentity = { ...identity, providerGeneration: 2, deliveryToken: 'token-2' }
const nextSpan = { ...span, ...nextIdentity, spanId: 'span-2' }
coordinator.open(nextIdentity)
coordinator.commit(coordinator.reserve(nextIdentity, nextSpan, ['model']))
coordinator.settle({
identity: nextIdentity,
spanId: nextSpan.spanId,
consumer: 'model',
reason: 'accepted'
})
coordinator.flushAcknowledgements()
expect(publish).toHaveBeenCalledWith(
2,
{
acknowledgements: [expect.objectContaining({ deliveryToken: 'token-2', creditedEndSu: 4 })]
},
expect.any(Function)
)
})
it('generation-closes every retained token on coordinator disposal', () => {
const coordinator = new SshPtySourceObligationCoordinator({
publish: vi.fn(),
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
cancelSchedule: vi.fn()
})
coordinator.open(identity)
coordinator.commit(coordinator.reserve(identity, span, ['model']))
coordinator.dispose('provider disposed')
expect(coordinator.snapshot(identity)).toMatchObject({
state: 'closed',
generationClosed: true,
openSpans: 0
})
expect(() => coordinator.open({ ...identity, deliveryToken: 'late-token' })).toThrow('disposed')
})
})
@@ -0,0 +1,267 @@
import {
ptySourceDeliveryKey,
samePtySourceDelivery,
type PtySourceDeliveryIdentity,
type PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import {
SshPtySourceAckCoalescer,
type SshPtySourceAckCoalescerOptions
} from './ssh-pty-source-ack-coalescer'
import {
SshPtySourceObligationLedger,
type SshPtySourceAdmissionReservation,
type SshPtySourceConsumerId,
type SshPtySourceObligationState,
type SshPtySourceTokenSnapshot
} from './ssh-pty-source-obligation-ledger'
export type SshPtySourceObligationTransition = Readonly<{
identity: PtySourceDeliveryIdentity
spanId: string
consumer: SshPtySourceConsumerId
reason: string
}>
type TerminalWaiter = {
resolve: () => void
reject: (error: Error) => void
}
type TerminalWaiterGroup = {
identity: PtySourceDeliveryIdentity
waiters: Set<TerminalWaiter>
}
export class SshPtySourceObligationCoordinator {
private readonly ledger: SshPtySourceObligationLedger
private readonly acknowledgements: SshPtySourceAckCoalescer
private readonly terminalWaiters = new Map<string, TerminalWaiterGroup>()
private disposed = false
constructor(options: SshPtySourceAckCoalescerOptions) {
this.ledger = new SshPtySourceObligationLedger(options.onTokenClosed)
this.acknowledgements = new SshPtySourceAckCoalescer(options)
}
open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void {
if (this.disposed) {
throw new Error('SSH PTY source obligation coordinator is disposed')
}
this.ledger.open(identity, checkpointSourceEndSu)
}
reserve(
identity: PtySourceDeliveryIdentity,
span: PtySourceSpan,
requiredConsumers: readonly SshPtySourceConsumerId[]
): SshPtySourceAdmissionReservation {
return this.ledger.reserve(identity, span, requiredConsumers)
}
commit(reservation: SshPtySourceAdmissionReservation): void {
this.ledger.commit(reservation)
}
rollback(reservation: SshPtySourceAdmissionReservation): boolean {
return this.ledger.rollback(reservation)
}
rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean {
const rolledBack = this.ledger.rollbackCommitted(reservation)
if (rolledBack) {
this.maybeResolveTerminal(reservation.span)
}
return rolledBack
}
settle(transition: SshPtySourceObligationTransition): boolean {
this.requireSpanIdentity(transition)
const changed = this.ledger.settle(transition.spanId, transition.consumer, transition.reason)
this.queueEligibleAck(transition.identity)
return changed
}
beginTransfer(transition: SshPtySourceObligationTransition, to: SshPtySourceConsumerId): boolean {
this.requireSpanIdentity(transition)
return this.ledger.beginTransfer(transition.spanId, transition.consumer, to, transition.reason)
}
commitTransfer(transition: Omit<SshPtySourceObligationTransition, 'reason'>): boolean {
this.requireSpanIdentity(transition)
const changed = this.ledger.commitTransfer(transition.spanId, transition.consumer)
this.queueEligibleAck(transition.identity)
return changed
}
cancelTransfer(transition: SshPtySourceObligationTransition): boolean {
this.requireSpanIdentity(transition)
const changed = this.ledger.cancelTransfer(
transition.spanId,
transition.consumer,
transition.reason
)
this.queueEligibleAck(transition.identity)
return changed
}
rollbackTransfer(transition: SshPtySourceObligationTransition): boolean {
this.requireSpanIdentity(transition)
return this.ledger.rollbackTransfer(transition.spanId, transition.consumer)
}
seal(identity: PtySourceDeliveryIdentity): void {
if (this.ledger.snapshot(identity).state === 'sealed-unsettled') {
return
}
this.ledger.seal(identity)
}
markExitPublished(identity: PtySourceDeliveryIdentity): void {
this.queueEligibleAck(identity)
this.ledger.markExitPublished(identity)
}
whenTerminal(identity: PtySourceDeliveryIdentity): Promise<void> {
if (this.isTerminal(identity)) {
return Promise.resolve()
}
const key = ptySourceDeliveryKey(identity)
let group = this.terminalWaiters.get(key)
if (!group) {
group = { identity: Object.freeze({ ...identity }), waiters: new Set() }
this.terminalWaiters.set(key, group)
}
return new Promise((resolve, reject) => {
group!.waiters.add({ resolve, reject })
})
}
beginExitTimeout(identity: PtySourceDeliveryIdentity) {
return this.ledger.beginExitTimeout(identity)
}
applyCancellationProof(
identity: PtySourceDeliveryIdentity,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
const snapshot = this.ledger.snapshot(identity)
if (
snapshot.state === 'closed' &&
snapshot.receivedEndSu === proof.sentEndSu &&
snapshot.ackPublishedEndSu === proof.creditedEndSu
) {
return
}
this.ledger.applyCancellationProof(identity, proof)
this.rejectWaiters(
(group) => samePtySourceDelivery(group.identity, identity),
new Error('ssh_source_delivery_canceled')
)
}
applyRecoveryCancellationProof(
identity: PtySourceDeliveryIdentity,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
this.ledger.applyRecoveryCancellationProof(identity, proof)
this.rejectWaiters(
(group) => samePtySourceDelivery(group.identity, identity),
new Error('ssh_source_delivery_canceled')
)
}
closeGeneration(providerGeneration: number, reason: string): number {
this.rejectWaiters(
(group) => group.identity.providerGeneration === providerGeneration,
new Error(reason)
)
const closed = this.ledger.closeGeneration(providerGeneration, reason)
this.acknowledgements.cancelGeneration(providerGeneration, reason)
return closed
}
snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot {
return this.ledger.snapshot(identity)
}
modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number {
return this.ledger.modelAcceptedEnd(identity)
}
obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState {
return this.ledger.obligation(spanId, consumer)
}
spanIdentity(spanId: string): PtySourceSpan {
return this.ledger.spanIdentity(spanId)
}
hasRetainedSpan(spanId: string): boolean {
return this.ledger.hasRetainedSpan(spanId)
}
flushAcknowledgements(): void {
this.acknowledgements.flush()
}
dispose(reason?: string): void {
if (this.disposed) {
return
}
this.disposed = true
this.rejectWaiters(() => true, new Error(reason ?? 'SSH PTY source obligations disposed'))
this.ledger.closeAll(reason ?? 'SSH PTY source obligation coordinator disposed')
this.acknowledgements.dispose(reason)
}
private queueEligibleAck(identity: PtySourceDeliveryIdentity): void {
const publication = this.ledger.queueAck(identity)
if (publication) {
this.acknowledgements.enqueue(publication)
}
this.maybeResolveTerminal(identity)
}
private isTerminal(identity: PtySourceDeliveryIdentity): boolean {
const snapshot = this.ledger.snapshot(identity)
return (
snapshot.obligationsTerminalEndSu === snapshot.receivedEndSu &&
snapshot.ackQueuedEndSu === snapshot.receivedEndSu
)
}
private maybeResolveTerminal(identity: PtySourceDeliveryIdentity): void {
if (!this.isTerminal(identity)) {
return
}
const group = this.terminalWaiters.get(ptySourceDeliveryKey(identity))
if (!group) {
return
}
this.terminalWaiters.delete(ptySourceDeliveryKey(identity))
for (const waiter of group.waiters) {
waiter.resolve()
}
}
private rejectWaiters(predicate: (group: TerminalWaiterGroup) => boolean, error: Error): void {
for (const [key, group] of this.terminalWaiters) {
if (!predicate(group)) {
continue
}
this.terminalWaiters.delete(key)
for (const waiter of group.waiters) {
waiter.reject(error)
}
}
}
private requireSpanIdentity(
transition: Pick<SshPtySourceObligationTransition, 'identity' | 'spanId'>
): void {
if (!samePtySourceDelivery(this.ledger.spanIdentity(transition.spanId), transition.identity)) {
throw new Error('SSH PTY source obligation transition has a stale delivery identity')
}
}
}
@@ -0,0 +1,305 @@
import { describe, expect, it } from 'vitest'
import type {
PtySourceDeliveryIdentity,
PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import { SshPtySourceObligationLedger } from './ssh-pty-source-obligation-ledger'
function identity(
deliveryToken = 'token-1',
overrides: Partial<PtySourceDeliveryIdentity> = {}
): PtySourceDeliveryIdentity {
return {
id: 'pty-1',
providerGeneration: 1,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken,
...overrides
}
}
function span(
owner: PtySourceDeliveryIdentity,
spanId: string,
sourceStartSu: number,
data: string
): PtySourceSpan {
return Object.freeze({
...owner,
spanId,
sourceStartSu,
sourceEndSu: sourceStartSu + data.length,
displayStart: sourceStartSu,
displayEnd: sourceStartSu + data.length,
data,
splittable: true,
transform: Object.freeze({
transformed: false,
rawLengthSu: data.length,
scalarSafe: true
})
})
}
function commitSpan(
ledger: SshPtySourceObligationLedger,
owner: PtySourceDeliveryIdentity,
sourceSpan: PtySourceSpan,
consumers: ('model' | 'desktop')[] = ['model', 'desktop']
) {
const reservation = ledger.reserve(owner, sourceSpan, consumers)
ledger.commit(reservation)
return reservation
}
describe('SshPtySourceObligationLedger', () => {
it('rolls back an uncommitted admission without consuming its source coordinate', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
const first = ledger.reserve(owner, span(owner, 'span-1', 0, 'abc'), ['model'])
expect(ledger.rollback(first)).toBe(true)
const retry = ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model'])
ledger.commit(retry)
expect(ledger.snapshot(owner).receivedEndSu).toBe(3)
})
it('rolls back a committed tail while every obligation is still open', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
const reservation = commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'))
expect(ledger.rollbackCommitted(reservation)).toBe(true)
expect(ledger.snapshot(owner)).toMatchObject({ receivedEndSu: 0, openSpans: 0 })
expect(() => ledger.spanIdentity('span-1')).toThrow('Unknown or reclaimed')
expect(
ledger.commit(ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model']))
).toBeUndefined()
})
it('keeps terminal, queued, and successfully published ACK ends independent', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
ledger.settle('span-1', 'model', 'emulator-receipt')
expect(ledger.snapshot(owner)).toMatchObject({
obligationsTerminalEndSu: 0,
ackQueuedEndSu: 0,
ackPublishedEndSu: 0
})
ledger.settle('span-1', 'desktop', 'renderer-parse')
expect(ledger.snapshot(owner)).toMatchObject({
obligationsTerminalEndSu: 4,
ackQueuedEndSu: 0,
ackPublishedEndSu: 0
})
const publication = ledger.queueAck(owner)!
expect(ledger.snapshot(owner)).toMatchObject({
obligationsTerminalEndSu: 4,
ackQueuedEndSu: 4,
ackPublishedEndSu: 0
})
publication.onSettled({ ok: false, error: new Error('write callback failed') })
expect(ledger.snapshot(owner)).toMatchObject({ ackQueuedEndSu: 4, ackPublishedEndSu: 0 })
ledger.retryQueuedAck(owner)!.onSettled({ ok: true })
expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 0 })
})
it('checkpoints only the contiguous model-settled prefix', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh'))
expect(ledger.modelAcceptedEnd(owner)).toBe(0)
ledger.settle('span-2', 'model', 'out-of-order')
expect(ledger.modelAcceptedEnd(owner)).toBe(0)
ledger.settle('span-1', 'model', 'emulator-receipt')
expect(ledger.modelAcceptedEnd(owner)).toBe(8)
})
it('continues checkpoints from the reclaimed ACK-published prefix', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh'))
ledger.settle('span-1', 'model', 'emulator-receipt')
ledger.settle('span-1', 'desktop', 'renderer-parse')
ledger.queueAck(owner)!.onSettled({ ok: true })
expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 1 })
expect(ledger.modelAcceptedEnd(owner)).toBe(4)
ledger.settle('span-2', 'model', 'emulator-receipt')
expect(ledger.modelAcceptedEnd(owner)).toBe(8)
})
it('requires an exact transfer fence before a desktop obligation becomes terminal', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'))
ledger.settle('span-1', 'model', 'emulator-receipt')
expect(ledger.beginTransfer('span-1', 'desktop', 'model', 'renderer-send-failed')).toBe(true)
expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(0)
expect(ledger.commitTransfer('span-1', 'desktop')).toBe(true)
expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(3)
expect(ledger.obligation('span-1', 'desktop')).toMatchObject({
state: 'transferred',
to: 'model'
})
})
it('keeps sealed exit state until final ACK publication succeeds', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
ledger.seal(owner)
expect(() => ledger.markExitPublished(owner)).toThrow('terminal ACK')
ledger.settle('tail', 'model', 'emulator-receipt')
const publication = ledger.queueAck(owner)!
ledger.markExitPublished(owner)
expect(ledger.snapshot(owner).state).toBe('sealed-unsettled')
publication.onSettled({ ok: true })
expect(ledger.snapshot(owner).state).toBe('closed')
})
it('publishes token cancellation intent before accepting timeout cleanup proof', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
ledger.seal(owner)
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })).toThrow()
expect(ledger.beginExitTimeout(owner)).toEqual({
id: owner.id,
deliveryToken: owner.deliveryToken,
clientGeneration: owner.clientGeneration,
ownerGeneration: owner.ownerGeneration
})
ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
})
it('ignores a late successful write callback after generation-close proof', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'), ['model'])
ledger.settle('span-1', 'model', 'accepted')
const publication = ledger.queueAck(owner)!
expect(ledger.closeGeneration(1, 'provider-closed')).toBe(1)
publication.onSettled({ ok: true })
expect(ledger.snapshot(owner)).toMatchObject({
state: 'closed',
generationClosed: true,
ackPublishedEndSu: 0
})
expect(() => ledger.obligation('span-1', 'model')).toThrow('reclaimed')
})
it('rejects stale generations, tokens, and non-contiguous source spans', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
expect(() =>
ledger.reserve(identity('stale'), span(identity('stale'), 'stale', 0, 'x'), ['model'])
).toThrow('stale')
expect(() => ledger.reserve(owner, span(owner, 'gap', 1, 'x'), ['model'])).toThrow(
'non-contiguous'
)
})
it('requires cancellation proof to match the exact local received and published ends', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
ledger.seal(owner)
ledger.beginExitTimeout(owner)
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 3, creditedEndSu: 0 })).toThrow(
'invalid'
)
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 1 })).toThrow(
'invalid'
)
})
it('applies recovery cancellation proof over a locally admitted prefix', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner, 4)
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 12, creditedEndSu: 4 })
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
})
it('rejects recovery cancellation proof that misses local intake state', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner, 4)
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
expect(() =>
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 7, creditedEndSu: 4 })
).toThrow('invalid')
expect(() =>
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 8, creditedEndSu: 5 })
).toThrow('invalid')
})
it('bounds closed-token tombstones and removes committed reservation indexes', () => {
const ledger = new SshPtySourceObligationLedger()
const owners = Array.from({ length: 300 }, (_, index) =>
identity(`token-${index}`, {
id: `pty-${index}`,
ptyIncarnation: `incarnation-${index}`
})
)
for (const [index, owner] of owners.entries()) {
ledger.open(owner)
commitSpan(ledger, owner, span(owner, `span-${index}`, 0, 'x'), ['model'])
ledger.closeGeneration(1, 'generation-closed')
}
expect(() => ledger.snapshot(owners[0])).toThrow('stale')
expect(ledger.snapshot(owners.at(-1)!)).toMatchObject({ state: 'closed' })
})
it('reclaims uncommitted reservations on exact cancellation proof', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
const reservation = ledger.reserve(owner, span(owner, 'pending', 0, 'x'), ['model'])
ledger.seal(owner)
ledger.beginExitTimeout(owner)
ledger.applyCancellationProof(owner, { sentEndSu: 0, creditedEndSu: 0 })
expect(ledger.rollback(reservation)).toBe(false)
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
})
it('rejects reopening a recently closed one-use token', () => {
const ledger = new SshPtySourceObligationLedger()
const owner = identity()
ledger.open(owner)
ledger.closeGeneration(owner.providerGeneration, 'closed')
expect(() => ledger.open(owner)).toThrow('already used')
})
})
@@ -0,0 +1,329 @@
import {
ptySourceDeliveryKey,
samePtySourceDelivery,
type PtySourceDeliveryIdentity,
type PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import {
assertNonNegativeSafeInteger,
assertPtySourceIdentity,
assertPtySourceSpan
} from '../../shared/pty-source-credit-validation'
import type {
SshPtySourceAckPublication,
SshPtySourceAdmissionReservation,
SshPtySourceConsumerId,
SshPtySourceObligationState,
SshPtySourceTokenSnapshot
} from './ssh-pty-source-obligation-contract'
import { createSshPtySourceAckPublication } from './ssh-pty-source-ack-publication'
import {
beginSourceExitTimeout,
cancelOpenSourceObligations,
closeAllSourceTokens,
closeSourceGeneration,
closeSourceToken,
createSourceSpanRecord,
createSourceToken,
markSourceExitPublished,
requireSourceSpan,
rollbackCommittedSourceSpan,
sealSourceToken,
snapshotSourceToken,
type ReservationRecord,
type TokenRecord
} from './ssh-pty-source-obligation-state'
import {
applySourceRecoveryCancellationProof,
cancelSourceObligationTransfer,
commitSourceObligationTransfer,
modelAcceptedSourceEnd,
rollbackSourceObligationTransfer,
transitionOpenSourceObligation
} from './ssh-pty-source-obligation-transitions'
export type {
SshPtySourceAckPublication,
SshPtySourceAdmissionReservation,
SshPtySourceConsumerId,
SshPtySourceObligationState,
SshPtySourceTokenSnapshot
} from './ssh-pty-source-obligation-contract'
export class SshPtySourceObligationLedger {
private readonly tokens = new Map<string, TokenRecord>()
private readonly closedSnapshots = new Map<string, SshPtySourceTokenSnapshot>()
private readonly reservations = new Map<string, ReservationRecord>()
private readonly spanOwners = new Map<string, TokenRecord>()
private nextReservationId = 1
constructor(
private readonly onTokenClosed: (identity: PtySourceDeliveryIdentity) => void = () => {}
) {}
open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void {
assertPtySourceIdentity(identity)
assertNonNegativeSafeInteger(checkpointSourceEndSu, 'checkpointSourceEndSu')
const key = ptySourceDeliveryKey(identity)
if (this.tokens.has(key) || this.closedSnapshots.has(key)) {
throw new Error('SSH PTY source token was already used')
}
this.tokens.set(key, createSourceToken(identity, checkpointSourceEndSu))
}
reserve(
identity: PtySourceDeliveryIdentity,
span: PtySourceSpan,
requiredConsumers: readonly SshPtySourceConsumerId[]
): SshPtySourceAdmissionReservation {
const token = this.requireToken(identity)
if (token.state !== 'active') {
throw new Error('SSH PTY source token no longer admits data')
}
assertPtySourceSpan(span)
if (
!samePtySourceDelivery(token.identity, span) ||
span.sourceStartSu !== token.receivedEndSu ||
this.spanOwners.has(span.spanId)
) {
throw new Error('SSH PTY source span is stale, duplicate, or non-contiguous')
}
const uniqueConsumers = Array.from(new Set(requiredConsumers))
if (!uniqueConsumers.includes('model')) {
throw new Error('SSH PTY source span requires the terminal model obligation')
}
const reservation = Object.freeze({
reservationId: `ssh-source-admission:${this.nextReservationId++}`,
span,
requiredConsumers: Object.freeze(uniqueConsumers)
})
this.reservations.set(reservation.reservationId, {
reservation,
state: 'reserved'
})
return reservation
}
commit(reservation: SshPtySourceAdmissionReservation): void {
const record = this.requireReservation(reservation)
if (record.state !== 'reserved') {
throw new Error('SSH PTY source admission reservation is not pending')
}
const token = this.requireToken(reservation.span)
if (token.state !== 'active' || token.receivedEndSu !== reservation.span.sourceStartSu) {
throw new Error('SSH PTY source admission reservation became stale')
}
const spanRecord = createSourceSpanRecord(reservation.span, reservation.requiredConsumers)
token.spans.push(spanRecord)
token.receivedEndSu = reservation.span.sourceEndSu
this.spanOwners.set(reservation.span.spanId, token)
this.reservations.delete(reservation.reservationId)
}
rollback(reservation: SshPtySourceAdmissionReservation): boolean {
const record = this.reservations.get(reservation.reservationId)
if (!record || record.reservation !== reservation || record.state !== 'reserved') {
return false
}
this.reservations.delete(reservation.reservationId)
return true
}
rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean {
const token = this.tokens.get(ptySourceDeliveryKey(reservation.span))
if (!token || !samePtySourceDelivery(token.identity, reservation.span)) {
return false
}
return rollbackCommittedSourceSpan(token, reservation, this.spanOwners)
}
settle(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean {
return this.transitionOpen(spanId, consumer, Object.freeze({ state: 'settled', reason }))
}
beginTransfer(
spanId: string,
consumer: SshPtySourceConsumerId,
to: SshPtySourceConsumerId,
reason: string
): boolean {
return this.transitionOpen(
spanId,
consumer,
Object.freeze({ state: 'transferring', to, reason })
)
}
commitTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean {
return commitSourceObligationTransfer(this.spanOwners, spanId, consumer)
}
cancelTransfer(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean {
return cancelSourceObligationTransfer(this.spanOwners, spanId, consumer, reason)
}
rollbackTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean {
return rollbackSourceObligationTransfer(this.spanOwners, spanId, consumer)
}
queueAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null {
const token = this.requireToken(identity)
if (token.obligationsTerminalEndSu <= token.ackQueuedEndSu) {
return null
}
const endSu = token.obligationsTerminalEndSu
token.ackQueuedEndSu = endSu
return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () =>
this.maybeClose(token)
)
}
retryQueuedAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null {
const token = this.requireToken(identity)
if (token.ackQueuedEndSu <= token.ackPublishedEndSu) {
return null
}
const endSu = token.ackQueuedEndSu
return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () =>
this.maybeClose(token)
)
}
seal(identity: PtySourceDeliveryIdentity): void {
sealSourceToken(this.requireToken(identity))
}
markExitPublished(identity: PtySourceDeliveryIdentity): void {
const token = this.requireToken(identity)
markSourceExitPublished(token)
this.maybeClose(token)
}
beginExitTimeout(identity: PtySourceDeliveryIdentity): Readonly<{
id: string
deliveryToken: string
clientGeneration: number
ownerGeneration: number
}> {
return beginSourceExitTimeout(this.requireToken(identity))
}
applyCancellationProof(
identity: PtySourceDeliveryIdentity,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
const token = this.requireToken(identity)
if (
token.state !== 'canceling' ||
proof.sentEndSu !== token.receivedEndSu ||
proof.creditedEndSu !== token.ackPublishedEndSu
) {
throw new Error('SSH PTY source cancellation proof is stale or invalid')
}
cancelOpenSourceObligations(token, 'relay-cancellation-proof')
this.closeToken(token)
}
applyRecoveryCancellationProof(
identity: PtySourceDeliveryIdentity,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
const token = this.requireToken(identity)
applySourceRecoveryCancellationProof(token, proof)
this.closeToken(token)
}
closeGeneration(providerGeneration: number, reason: string): number {
return closeSourceGeneration(
this.tokens,
this.reservations,
providerGeneration,
reason,
(token) => this.closeToken(token)
)
}
closeAll(reason: string): number {
return closeAllSourceTokens(this.tokens, this.reservations, reason, (token) =>
this.closeToken(token)
)
}
snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot {
const key = ptySourceDeliveryKey(identity)
const token = this.tokens.get(key)
if (token && samePtySourceDelivery(token.identity, identity)) {
return snapshotSourceToken(token)
}
const closed = this.closedSnapshots.get(key)
if (closed && samePtySourceDelivery(closed, identity)) {
return closed
}
throw new Error('Unknown or stale SSH PTY source token')
}
modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number {
return modelAcceptedSourceEnd(this.requireToken(identity))
}
obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState {
const obligation = requireSourceSpan(this.spanOwners, spanId).span.obligations.get(consumer)
if (!obligation) {
throw new Error('SSH PTY source consumer obligation does not exist')
}
return obligation
}
spanIdentity(spanId: string): PtySourceSpan {
return requireSourceSpan(this.spanOwners, spanId).span.span
}
hasRetainedSpan(spanId: string): boolean {
return this.spanOwners.has(spanId)
}
private transitionOpen(
spanId: string,
consumer: SshPtySourceConsumerId,
next: SshPtySourceObligationState
): boolean {
return transitionOpenSourceObligation(this.spanOwners, spanId, consumer, next)
}
private maybeClose(token: TokenRecord): void {
if (
token.state === 'sealed-unsettled' &&
token.exitPublished &&
token.ackPublishedEndSu === token.receivedEndSu
) {
this.closeToken(token)
}
}
private closeToken(token: TokenRecord): void {
closeSourceToken(
token,
this.tokens,
this.closedSnapshots,
this.reservations,
this.spanOwners,
this.onTokenClosed
)
}
private requireReservation(reservation: SshPtySourceAdmissionReservation): ReservationRecord {
const record = this.reservations.get(reservation.reservationId)
if (!record || record.reservation !== reservation) {
throw new Error('Unknown SSH PTY source admission reservation')
}
return record
}
private requireToken(identity: PtySourceDeliveryIdentity): TokenRecord {
const token = this.tokens.get(ptySourceDeliveryKey(identity))
if (!token || !samePtySourceDelivery(token.identity, identity)) {
throw new Error('Unknown or stale SSH PTY source token')
}
return token
}
}
@@ -0,0 +1,287 @@
import type {
PtySourceDeliveryIdentity,
PtySourceSpan
} from '../../shared/pty-source-credit-contract'
import {
ptySourceDeliveryKey,
samePtySourceDelivery
} from '../../shared/pty-source-credit-contract'
import type {
SshPtySourceAdmissionReservation,
SshPtySourceConsumerId,
SshPtySourceObligationState,
SshPtySourceTokenSnapshot
} from './ssh-pty-source-obligation-contract'
export const CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT = 256
export type SpanRecord = {
span: PtySourceSpan
obligations: Map<SshPtySourceConsumerId, SshPtySourceObligationState>
}
export type ReservationRecord = {
reservation: SshPtySourceAdmissionReservation
state: 'reserved' | 'committed' | 'rolled-back'
}
export type TokenRecord = {
identity: PtySourceDeliveryIdentity
state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed'
checkpointSourceEndSu: number
receivedEndSu: number
obligationsTerminalEndSu: number
ackQueuedEndSu: number
ackPublishedEndSu: number
spans: SpanRecord[]
exitPublished: boolean
generationClosed: boolean
}
export function createSourceToken(
identity: PtySourceDeliveryIdentity,
checkpointSourceEndSu: number
): TokenRecord {
return {
identity: Object.freeze({ ...identity }),
state: 'active',
checkpointSourceEndSu,
receivedEndSu: checkpointSourceEndSu,
obligationsTerminalEndSu: checkpointSourceEndSu,
ackQueuedEndSu: checkpointSourceEndSu,
ackPublishedEndSu: checkpointSourceEndSu,
spans: [],
exitPublished: false,
generationClosed: false
}
}
export function createSourceSpanRecord(
span: PtySourceSpan,
consumers: readonly SshPtySourceConsumerId[]
): SpanRecord {
return {
span,
obligations: new Map(
consumers.map((consumer) => [consumer, Object.freeze({ state: 'open' as const })])
)
}
}
export function sealSourceToken(token: TokenRecord): void {
if (token.state !== 'active') {
throw new Error('SSH PTY source token cannot be sealed from its current state')
}
token.state = 'sealed-unsettled'
}
export function markSourceExitPublished(token: TokenRecord): void {
if (token.state !== 'sealed-unsettled') {
throw new Error('SSH PTY source exit publication requires a sealed token')
}
if (
token.obligationsTerminalEndSu !== token.receivedEndSu ||
token.ackQueuedEndSu !== token.receivedEndSu
) {
throw new Error('SSH PTY source exit cannot publish before terminal ACK queueing')
}
token.exitPublished = true
}
export function beginSourceExitTimeout(token: TokenRecord): Readonly<{
id: string
deliveryToken: string
clientGeneration: number
ownerGeneration: number
}> {
if (token.state !== 'sealed-unsettled') {
throw new Error('SSH PTY source exit timeout requires a sealed token')
}
token.state = 'canceling'
return Object.freeze({
id: token.identity.id,
deliveryToken: token.identity.deliveryToken,
clientGeneration: token.identity.clientGeneration,
ownerGeneration: token.identity.ownerGeneration
})
}
function obligationIsTerminal(obligation: SshPtySourceObligationState): boolean {
return (
obligation.state === 'settled' ||
obligation.state === 'transferred' ||
obligation.state === 'canceled'
)
}
export function snapshotSourceToken(token: TokenRecord): SshPtySourceTokenSnapshot {
return Object.freeze({
...token.identity,
state: token.state,
receivedEndSu: token.receivedEndSu,
obligationsTerminalEndSu: token.obligationsTerminalEndSu,
ackQueuedEndSu: token.ackQueuedEndSu,
ackPublishedEndSu: token.ackPublishedEndSu,
openSpans: token.spans.length,
exitPublished: token.exitPublished,
generationClosed: token.generationClosed
})
}
export function advanceSourceTerminalEnd(token: TokenRecord): void {
let endSu = token.obligationsTerminalEndSu
for (const record of token.spans) {
if (record.span.sourceEndSu <= endSu) {
continue
}
if (
record.span.sourceStartSu !== endSu ||
!Array.from(record.obligations.values()).every(obligationIsTerminal)
) {
break
}
endSu = record.span.sourceEndSu
}
token.obligationsTerminalEndSu = endSu
}
export function cancelOpenSourceObligations(token: TokenRecord, reason: string): void {
for (const record of token.spans) {
for (const [consumer, obligation] of record.obligations) {
if (obligation.state === 'open' || obligation.state === 'transferring') {
record.obligations.set(consumer, Object.freeze({ state: 'canceled', reason }))
}
}
}
advanceSourceTerminalEnd(token)
}
export function reclaimPublishedSourcePrefix(
token: TokenRecord,
spanOwners: Map<string, TokenRecord>
): void {
while (token.spans[0]?.span.sourceEndSu <= token.ackPublishedEndSu) {
const record = token.spans.shift()!
spanOwners.delete(record.span.spanId)
}
}
export function releaseSourceTokenSpans(
token: TokenRecord,
spanOwners: Map<string, TokenRecord>
): void {
for (const record of token.spans) {
spanOwners.delete(record.span.spanId)
}
token.spans = []
}
export function releaseSourceTokenReservations(
token: TokenRecord,
reservations: Map<string, ReservationRecord>
): void {
for (const [reservationId, record] of reservations) {
if (samePtySourceDelivery(record.reservation.span, token.identity)) {
record.state = 'rolled-back'
reservations.delete(reservationId)
}
}
}
export function rollbackCommittedSourceSpan(
token: TokenRecord,
reservation: SshPtySourceAdmissionReservation,
spanOwners: Map<string, TokenRecord>
): boolean {
const last = token.spans.at(-1)
if (
last?.span !== reservation.span ||
token.receivedEndSu !== reservation.span.sourceEndSu ||
token.obligationsTerminalEndSu > reservation.span.sourceStartSu ||
token.ackQueuedEndSu > reservation.span.sourceStartSu ||
Array.from(last.obligations.values()).some((obligation) => obligation.state !== 'open')
) {
return false
}
token.spans.pop()
token.receivedEndSu = reservation.span.sourceStartSu
spanOwners.delete(reservation.span.spanId)
return true
}
export function requireSourceSpan(
spanOwners: ReadonlyMap<string, TokenRecord>,
spanId: string
): { token: TokenRecord; span: SpanRecord } {
const token = spanOwners.get(spanId)
const span = token?.spans.find((candidate) => candidate.span.spanId === spanId)
if (!token || !span) {
throw new Error('Unknown or reclaimed SSH PTY source span')
}
return { token, span }
}
export function closeSourceGeneration(
tokens: Map<string, TokenRecord>,
reservations: Map<string, ReservationRecord>,
providerGeneration: number,
reason: string,
closeToken: (token: TokenRecord) => void
): number {
let closed = 0
for (const token of Array.from(tokens.values())) {
if (token.identity.providerGeneration !== providerGeneration || token.state === 'closed') {
continue
}
token.generationClosed = true
cancelOpenSourceObligations(token, reason)
closeToken(token)
closed++
}
for (const [id, record] of reservations) {
if (
record.state === 'reserved' &&
record.reservation.span.providerGeneration === providerGeneration
) {
record.state = 'rolled-back'
reservations.delete(id)
}
}
return closed
}
export function closeAllSourceTokens(
tokens: Map<string, TokenRecord>,
reservations: Map<string, ReservationRecord>,
reason: string,
closeToken: (token: TokenRecord) => void
): number {
let closed = 0
const generations = new Set(
Array.from(tokens.values(), (token) => token.identity.providerGeneration)
)
for (const providerGeneration of generations) {
closed += closeSourceGeneration(tokens, reservations, providerGeneration, reason, closeToken)
}
return closed
}
export function closeSourceToken(
token: TokenRecord,
tokens: Map<string, TokenRecord>,
closedSnapshots: Map<string, SshPtySourceTokenSnapshot>,
reservations: Map<string, ReservationRecord>,
spanOwners: Map<string, TokenRecord>,
onTokenClosed: (identity: PtySourceDeliveryIdentity) => void
): void {
token.state = 'closed'
releaseSourceTokenSpans(token, spanOwners)
releaseSourceTokenReservations(token, reservations)
const key = ptySourceDeliveryKey(token.identity)
tokens.delete(key)
closedSnapshots.set(key, snapshotSourceToken(token))
while (closedSnapshots.size > CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT) {
closedSnapshots.delete(closedSnapshots.keys().next().value!)
}
onTokenClosed(token.identity)
}
@@ -0,0 +1,100 @@
import type {
SshPtySourceConsumerId,
SshPtySourceObligationState
} from './ssh-pty-source-obligation-contract'
import {
advanceSourceTerminalEnd,
cancelOpenSourceObligations,
requireSourceSpan,
type TokenRecord
} from './ssh-pty-source-obligation-state'
export function applySourceRecoveryCancellationProof(
token: TokenRecord,
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
): void {
if (
token.state !== 'active' ||
proof.sentEndSu < token.receivedEndSu ||
proof.creditedEndSu !== token.ackPublishedEndSu ||
proof.creditedEndSu > token.receivedEndSu
) {
throw new Error('SSH PTY source recovery cancellation proof is stale or invalid')
}
cancelOpenSourceObligations(token, 'relay-recovery-cancellation-proof')
}
export function transitionOpenSourceObligation(
spanOwners: ReadonlyMap<string, TokenRecord>,
spanId: string,
consumer: SshPtySourceConsumerId,
next: SshPtySourceObligationState
): boolean {
const { token, span } = requireSourceSpan(spanOwners, spanId)
if (span.obligations.get(consumer)?.state !== 'open') {
return false
}
span.obligations.set(consumer, next)
advanceSourceTerminalEnd(token)
return true
}
export function commitSourceObligationTransfer(
spanOwners: ReadonlyMap<string, TokenRecord>,
spanId: string,
consumer: SshPtySourceConsumerId
): boolean {
const { token, span } = requireSourceSpan(spanOwners, spanId)
const current = span.obligations.get(consumer)
if (current?.state !== 'transferring') {
return false
}
span.obligations.set(
consumer,
Object.freeze({ state: 'transferred', to: current.to, reason: current.reason })
)
advanceSourceTerminalEnd(token)
return true
}
export function cancelSourceObligationTransfer(
spanOwners: ReadonlyMap<string, TokenRecord>,
spanId: string,
consumer: SshPtySourceConsumerId,
reason: string
): boolean {
const { token, span } = requireSourceSpan(spanOwners, spanId)
if (span.obligations.get(consumer)?.state !== 'transferring') {
return false
}
span.obligations.set(consumer, Object.freeze({ state: 'canceled', reason }))
advanceSourceTerminalEnd(token)
return true
}
export function rollbackSourceObligationTransfer(
spanOwners: ReadonlyMap<string, TokenRecord>,
spanId: string,
consumer: SshPtySourceConsumerId
): boolean {
const { span } = requireSourceSpan(spanOwners, spanId)
if (span.obligations.get(consumer)?.state !== 'transferring') {
return false
}
span.obligations.set(consumer, Object.freeze({ state: 'open' }))
return true
}
export function modelAcceptedSourceEnd(token: TokenRecord): number {
let acceptedEndSu = token.ackPublishedEndSu
for (const record of token.spans) {
if (
record.span.sourceStartSu !== acceptedEndSu ||
record.obligations.get('model')?.state !== 'settled'
) {
break
}
acceptedEndSu = record.span.sourceEndSu
}
return acceptedEndSu
}
+132 -45
View File
@@ -11,6 +11,8 @@ const {
mockConnectionManager,
mockDeployAndLaunchRelay,
mockForceStopRelayForTarget,
mockAcceptSshPtyOutputData,
mockAcceptSshPtyOutputExit,
mockMux,
mockPtyProvider,
mockFsProvider,
@@ -49,10 +51,13 @@ const {
},
mockDeployAndLaunchRelay: vi.fn(),
mockForceStopRelayForTarget: vi.fn(),
mockAcceptSshPtyOutputData: vi.fn().mockResolvedValue({}),
mockAcceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined),
mockMux: {
dispose: vi.fn(),
isDisposed: vi.fn().mockReturnValue(false),
onNotification: vi.fn(),
onNotificationByMethod: vi.fn().mockReturnValue(() => {}),
onRequest: vi.fn().mockReturnValue(() => {}),
onDispose: vi.fn().mockReturnValue(() => {}),
request: vi.fn().mockResolvedValue({}),
@@ -65,7 +70,8 @@ const {
onReplay: vi.fn(),
attach: vi.fn(),
attachForReconnect: vi.fn().mockResolvedValue({}),
shutdown: vi.fn()
shutdown: vi.fn(),
providerGeneration: 0
},
mockFsProvider: {},
mockGitProvider: {},
@@ -100,6 +106,25 @@ vi.mock('electron', () => ({
}
}))
vi.mock('./ssh-pty-output-intake-registry', () => ({
acceptSshPtyOutputData: mockAcceptSshPtyOutputData,
acceptSshPtyOutputExit: mockAcceptSshPtyOutputExit,
allocateSshPtyProviderGeneration: (() => {
let generation = 0
return () => ++generation
})(),
beginSshPtyOutputGenerationMigration: vi.fn(() => ({
byPty: new Map(),
completion: Promise.resolve()
})),
applySshPtySourceCancellationProof: vi.fn().mockReturnValue(false),
applySshPtySourceRecoveryCancellationProof: vi.fn().mockReturnValue(false),
closeSshPtyOutputGeneration: vi.fn(),
getSshPtyAcceptedSourceCheckpoints: vi.fn().mockReturnValue([]),
installSshPtySourceAckPublisher: vi.fn().mockReturnValue(() => {}),
installSshPtySourceCancellationPublisher: vi.fn().mockReturnValue(() => {})
}))
vi.mock('../ssh/ssh-connection-store', () => ({
SshConnectionStore: class MockSshConnectionStore {
constructor() {
@@ -142,7 +167,8 @@ vi.mock('../providers/ssh-pty-provider', () => ({
isSshPtyNotFoundError: (err: unknown) =>
(err instanceof Error ? err.message : String(err)).includes('not found'),
SshPtyProvider: class MockSshPtyProvider {
constructor() {
constructor(_targetId: unknown, _mux: unknown, _env: unknown, providerGeneration: number) {
mockPtyProvider.providerGeneration = providerGeneration
return mockPtyProvider
}
}
@@ -222,6 +248,9 @@ import {
type SshConnectionState,
type SshTarget
} from '../../shared/ssh-types'
import { PTY_CONSUMER_SESSION_PROTOCOL_VERSION } from '../../shared/pty-consumer-session'
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
import {
clearProviderPtyState,
deletePtyOwnership,
@@ -231,6 +260,16 @@ import {
import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation'
describe('SSH IPC handlers', () => {
const relayBuildId = '0.1.0+ipc-test'
const ipcTestSource = {
relayPtyId: 'remote-pty',
spanId: 'ipc-test-delivery:0:5',
clientGeneration: 1,
ownerGeneration: 1,
deliveryToken: 'ipc-test-delivery',
sourceStartSu: 0,
sourceEndSu: 5
} as const
const handlers = new Map<string, (_event: unknown, args: unknown) => unknown>()
const mockStore = {
getRepos: () => [],
@@ -272,7 +311,8 @@ describe('SSH IPC handlers', () => {
const relayLostStabilizedMs = 5_000
const createRelayLaunchResult = () => ({
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
platform: 'linux-x64'
platform: 'linux-x64',
serverBuildId: relayBuildId
})
const getLatestRelayDisposeCallback = (): RelayDisposeCallback => {
const calls = mockMux.onDispose.mock.calls
@@ -322,21 +362,43 @@ describe('SSH IPC handlers', () => {
mockConnectionManager.setCallbacks.mockReset()
mockConnectionManager.callbacksRef.current = null
mockForceStopRelayForTarget.mockReset().mockResolvedValue(undefined)
mockAcceptSshPtyOutputData.mockReset().mockResolvedValue({})
mockAcceptSshPtyOutputExit.mockReset().mockResolvedValue(undefined)
mockDeployAndLaunchRelay.mockReset().mockResolvedValue({
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
platform: 'linux-x64'
platform: 'linux-x64',
serverBuildId: relayBuildId
})
mockMux.dispose.mockReset()
mockMux.isDisposed.mockReset().mockReturnValue(false)
mockMux.onNotification.mockReset()
mockMux.onNotificationByMethod.mockReset().mockReturnValue(() => {})
mockMux.onDispose.mockReset().mockReturnValue(() => {})
mockMux.request.mockReset().mockImplementation((method: string) =>
Promise.resolve(
method === 'pty.openClient'
? {
protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
serverBuildId: relayBuildId,
clientGeneration: 1,
role: 'session-owner',
ownerGeneration: 1,
ownerLease: 'ipc-test-owner',
capabilities: {
outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
}
}
: {}
)
)
mockMux.probeLiveness.mockReset().mockResolvedValue(false)
mockPtyProvider.onData.mockReset()
mockPtyProvider.onExit.mockReset()
mockPtyProvider.onReplay.mockReset()
mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({})
mockPtyProvider.shutdown.mockReset()
mockPtyProvider.providerGeneration = 0
mockRegisterSshGitProvider.mockReset()
mockPortForwardManager.addForward.mockReset()
mockPortForwardManager.updateForward.mockReset()
@@ -565,6 +627,7 @@ describe('SSH IPC handlers', () => {
}
mockDeployAndLaunchRelay.mockResolvedValueOnce({
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
serverBuildId: relayBuildId,
hostPlatform
})
mockSshStore.getTarget.mockReturnValue(target)
@@ -1050,7 +1113,7 @@ describe('SSH IPC handlers', () => {
}
})
it('forwards remote PTY events into the runtime', async () => {
it('forwards remote PTY events through the output intake authority', async () => {
const runtime = {
onPtyData: vi.fn(),
onPtyExit: vi.fn()
@@ -1073,24 +1136,47 @@ describe('SSH IPC handlers', () => {
})
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as
| ((payload: { id: string; data: string }) => void)
| undefined
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined
const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as
| ((payload: { id: string; code: number }) => void)
| ((payload: {
id: string
code: number
providerGeneration: number
ptyIncarnation: string
}) => void)
| undefined
onData?.({ id: 'remote-pty', data: 'hello' })
onExit?.({ id: 'remote-pty', code: 7 })
onData?.({
id: 'remote-pty',
data: 'hello',
providerGeneration: mockPtyProvider.providerGeneration,
ptyIncarnation: 'ipc-test-pty',
source: ipcTestSource
})
onExit?.({
id: 'remote-pty',
code: 7,
providerGeneration: mockPtyProvider.providerGeneration,
ptyIncarnation: 'ipc-test-pty'
})
expect(runtime.onPtyData).toHaveBeenCalledWith(
'remote-pty',
'hello',
expect.any(Number),
'hello'.length,
undefined
)
expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 7, undefined)
expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith({
id: 'remote-pty',
data: 'hello',
providerGeneration: mockPtyProvider.providerGeneration,
ptyIncarnation: 'ipc-test-pty',
rawLength: 'hello'.length,
transformed: false,
source: ipcTestSource
})
expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith({
id: 'remote-pty',
code: 7,
providerGeneration: mockPtyProvider.providerGeneration,
ptyIncarnation: 'ipc-test-pty'
})
expect(runtime.onPtyData).not.toHaveBeenCalled()
expect(runtime.onPtyExit).not.toHaveBeenCalled()
})
it('mirrors SSH state broadcasts onto the runtime client-event stream', async () => {
@@ -1380,7 +1466,7 @@ describe('SSH IPC handlers', () => {
expect(replacementConnectionManager.disconnect).not.toHaveBeenCalled()
})
it('refreshes live session callbacks to the newest window, store, and runtime', async () => {
it('refreshes live session callbacks to the newest window and output authorities', async () => {
const firstWindow = createMockWindow()
const secondWindow = createMockWindow()
const firstRuntime = {
@@ -1411,11 +1497,14 @@ describe('SSH IPC handlers', () => {
})
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as
| ((payload: { id: string; data: string }) => void)
| undefined
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined
const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as
| ((payload: { id: string; code: number }) => void)
| ((payload: {
id: string
code: number
providerGeneration: number
ptyIncarnation: string
}) => void)
| undefined
const onDetectedPorts = mockPortScannerCallbacks.get('ssh-1') as
| ((targetId: string, ports: unknown[], platform: string) => void)
@@ -1434,8 +1523,19 @@ describe('SSH IPC handlers', () => {
error: 'network down',
reconnectAttempt: 0
})
onData?.({ id: 'remote-pty', data: 'hello' })
onExit?.({ id: 'remote-pty', code: 9 })
onData?.({
id: 'remote-pty',
data: 'hello',
providerGeneration: mockPtyProvider.providerGeneration,
ptyIncarnation: 'ipc-test-pty',
source: ipcTestSource
})
onExit?.({
id: 'remote-pty',
code: 9,
providerGeneration: mockPtyProvider.providerGeneration,
ptyIncarnation: 'ipc-test-pty'
})
onDetectedPorts?.(
'ssh-1',
[{ host: '127.0.0.1', port: 3000, pid: 12, processName: 'node' }],
@@ -1454,33 +1554,20 @@ describe('SSH IPC handlers', () => {
connectionGeneration: 1
}
})
expect(secondWindow.webContents.send).toHaveBeenCalledWith(
'pty:data',
expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith(
expect.objectContaining({ id: 'remote-pty', data: 'hello' })
)
expect(secondWindow.webContents.send).toHaveBeenCalledWith('pty:exit', {
id: 'remote-pty',
code: 9
})
expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith(
expect.objectContaining({ id: 'remote-pty', code: 9 })
)
expect(secondWindow.webContents.send).toHaveBeenCalledWith('ssh:detected-ports-changed', {
targetId: 'ssh-1',
ports: expect.arrayContaining([expect.objectContaining({ port: 3000 })])
})
expect(secondRuntime.onPtyData).toHaveBeenCalledWith(
'remote-pty',
'hello',
expect.any(Number),
'hello'.length,
undefined
)
expect(secondRuntime.onPtyExit).toHaveBeenCalledWith('remote-pty', 9, undefined)
expect(secondRuntime.onPtyData).not.toHaveBeenCalled()
expect(secondRuntime.onPtyExit).not.toHaveBeenCalled()
expect(firstRuntime.onPtyData).not.toHaveBeenCalled()
expect(firstRuntime.onPtyExit).not.toHaveBeenCalled()
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
'ssh-1',
'remote-pty',
'terminated'
)
})
it('re-registers without replacing managers when no targets are connected', () => {
+20 -1
View File
@@ -1548,6 +1548,26 @@ describe('Store', () => {
expect(updatedTarget).not.toHaveProperty('systemSshConnectionReuse')
})
it('drops retired per-target SSH terminal source-credit selections', async () => {
const store = await createStore()
store.addSshTarget({
id: 'ssh-source-credit-on',
label: 'Noisy build host',
host: 'build.example.com',
port: 22,
username: 'dev',
experimentalPtySourceCreditV1: true
} as never)
expect(store.getSshTarget('ssh-source-credit-on')).not.toHaveProperty(
'experimentalPtySourceCreditV1'
)
store.flush()
const persisted = readDataFile() as { sshTargets?: Record<string, unknown>[] }
const target = persisted.sshTargets?.find((entry) => entry.id === 'ssh-source-credit-on')
expect(target).not.toHaveProperty('experimentalPtySourceCreditV1')
})
it('upserts ~/.ssh/config through the real store: rotated port updates in place and persists', async () => {
loadUserSshConfigMock.mockReturnValue([{ host: 'cluster' }])
const candidate = (port: number, id: string) => [
@@ -1563,7 +1583,6 @@ describe('Store', () => {
expect(inserted).toHaveLength(1)
expect(inserted[0]?.source).toBe('ssh-config')
expect(inserted[0]?.port).toBe(2200)
// Rotated port: upsert updates the same target in place and normalizeSshTarget must keep `source` (no false re-derive into a permanently-dirty state).
sshConfigHostsToTargetsMock.mockReturnValue(candidate(2222, 'ssh-cfg-2'))
const changed = sshStore.importFromSshConfig()
+2
View File
@@ -1113,6 +1113,7 @@ function backfillLegacyAutomationContexts(
type LegacySshTarget = SshTarget & {
remoteWorkspaceSyncEnabled?: unknown
remoteWorkspaceSyncGracePeriodSeconds?: unknown
experimentalPtySourceCreditV1?: unknown
}
// Why: old targets predate configHost; default to label-based lookup so imported SSH aliases still resolve via ssh -G.
@@ -1127,6 +1128,7 @@ function normalizeSshTarget(t: SshTarget): SshTarget {
delete target.remoteWorkspaceSyncGracePeriodSeconds
delete target.relayGracePeriodSeconds
delete target.systemSshConnectionReuse
delete target.experimentalPtySourceCreditV1
// Why: prefer the synced grace over stale relayGracePeriodSeconds so a user's "unlimited" (0) survives migration.
const relayGracePeriodSeconds =
legacySyncEnabled === true && typeof legacyGracePeriodSeconds === 'number'
+3
View File
@@ -2,6 +2,7 @@ import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges
import type { TuiAgent } from '../../shared/types'
import type { AgentSessionClaimedSpawnResult } from '../../shared/agent-session-host-authority'
import type { PtyIncarnationId } from '../../shared/pty-incarnation'
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
export type PtySpawnResult = {
agentSessionEnsure?: AgentSessionClaimedSpawnResult
@@ -10,6 +11,8 @@ export type PtySpawnResult = {
id: string
/** Opaque provider-owned identity for this process behind a reusable PTY id. */
incarnationId?: PtyIncarnationId
/** Relay source identity installed before adjacent source frames are decoded. */
sourceActivation?: PtySourceReceivingActivation
/** The provider observed this exact spawn exit before its control reply settled. */
exitedBeforeSpawnReply?: true
/** OS-level pid of the shell process, when available at spawn time.
@@ -2,6 +2,14 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION } from '../../shared/agent-session-host-authority'
import { isPtyIncarnationId } from '../../shared/pty-incarnation'
import type { PtySpawnResult } from './pty-spawn-result'
import type { PtySpawnOptions } from './types'
import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing'
import {
parsePtySourceReceivingActivation,
type PtySourceReceivingActivation
} from '../../shared/pty-source-receiving-activation'
import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation'
export const SSH_AGENT_SESSION_CAPABILITY_PROBE_TIMEOUT_MS = 5_000
@@ -49,11 +57,14 @@ export async function requestSshAgentSessionCreate(args: {
params: Record<string, unknown>
operationId?: string
signal?: AbortSignal
beforeResolve?: (result: unknown) => void
}): Promise<unknown> {
try {
return await (args.signal
? args.mux.request('pty.spawn', args.params, { signal: args.signal })
: args.mux.request('pty.spawn', args.params))
const options =
args.signal || args.beforeResolve
? { signal: args.signal, beforeResolve: args.beforeResolve }
: undefined
return await args.mux.request('pty.spawn', args.params, options)
} catch (error) {
if (!args.operationId) {
throw error
@@ -63,3 +74,111 @@ export async function requestSshAgentSessionCreate(args: {
throw Object.assign(spawnError, { agentSessionOperationOutcome: 'unknown' as const })
}
}
export async function spawnFreshSshPty(args: {
mux: SshChannelMultiplexer
options: PtySpawnOptions
params: Record<string, unknown>
exitRaceTracker: SshPtySpawnExitRaceTracker
installSourceActivation: (
relayPtyId: string,
activation: PtySourceReceivingActivation
) => SshPtyReceivingActivationLease
rememberPtyIncarnation: (relayPtyId: string, incarnationId: unknown) => void
acceptLivePty: (appPtyId: string) => void
toAppPtyId: (relayPtyId: string) => string
}): Promise<PtySpawnResult> {
const operation = args.exitRaceTracker.begin()
let sourceActivationLease: SshPtyReceivingActivationLease | undefined
try {
const result = await requestSshAgentSessionCreate({
mux: args.mux,
operationId: args.options.agentSessionCreateOperationId,
signal: args.options.signal,
params: args.params,
beforeResolve: (value) => {
sourceActivationLease = installSpawnSourceActivation(value, args.installSourceActivation)
}
})
if (args.options.agentSessionCreateOperationId) {
assertSshAgentSessionCreateResult(result)
}
const spawnResult = parseSshPtySpawnResult(result)
if (args.exitRaceTracker.didMatchingExitArrive(operation, spawnResult)) {
throw Object.assign(new Error('agent_session_exited_during_start'), {
agentSessionOperationOutcome: 'unknown' as const
})
}
const claimed = spawnResult.agentSessionEnsure
if (args.options.agentSessionEnsure) {
const validation = validateClaimedSshSpawn(spawnResult, args.options.agentSessionEnsure)
if (!validation.valid) {
if (validation.cleanup === 'created' && typeof spawnResult.id === 'string') {
try {
await args.mux.request('pty.shutdown', { id: spawnResult.id, immediate: true })
} catch {
throw new Error('execution_owner_unavailable')
}
}
throw new Error(validation.error)
}
}
const id = args.toAppPtyId(spawnResult.id)
args.rememberPtyIncarnation(spawnResult.id, spawnResult.incarnationId)
args.acceptLivePty(id)
const mappedResult = {
...spawnResult,
id,
...(claimed
? {
agentSessionEnsure: {
...claimed,
owner: { ...claimed.owner, ptyId: args.toAppPtyId(claimed.owner.ptyId) }
}
}
: {})
}
sourceActivationLease?.commit()
return mappedResult
} catch (error) {
if (sourceActivationLease && !(await sourceActivationLease.rollback())) {
throw new Error('execution_owner_unavailable')
}
throw error
} finally {
args.exitRaceTracker.finish(operation)
}
}
function installSpawnSourceActivation(
value: unknown,
install: (
relayPtyId: string,
activation: PtySourceReceivingActivation
) => SshPtyReceivingActivationLease
): SshPtyReceivingActivationLease | undefined {
const result = parseSshPtySpawnResult(value)
const activation = result.sourceActivation
if (!activation) {
return undefined
}
return install(result.id, activation)
}
function parseSshPtySpawnResult(value: unknown): PtySpawnResult {
const result =
typeof value === 'object' && value !== null && !Array.isArray(value)
? (value as PtySpawnResult)
: ({} as PtySpawnResult)
const activation = parsePtySourceReceivingActivation(result.sourceActivation)
if (
activation &&
(typeof result.id !== 'string' ||
result.id.length === 0 ||
!isPtyIncarnationId(result.incarnationId) ||
activation.ptyIncarnation !== result.incarnationId)
) {
throw new Error('Invalid SSH PTY source activation identity')
}
return activation ? { ...result, sourceActivation: activation } : result
}
@@ -1,13 +1,16 @@
import { describe, expect, it, vi } from 'vitest'
import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing'
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
type MockMux = {
onNotification: ReturnType<typeof vi.fn>
request: ReturnType<typeof vi.fn>
}
function createSubscription() {
const mux: MockMux = {
onNotification: vi.fn()
onNotification: vi.fn(),
request: vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
}
const dataListeners = new Set<(payload: { id: string; data: string }) => void>()
const replayListeners = new Set<(payload: { id: string; data: string }) => void>()
@@ -15,15 +18,18 @@ function createSubscription() {
const livePtyIds = new Set<string>()
const recordExit = vi.fn()
const toAppPtyId = vi.fn((id: string) => `ssh:conn@@${id}`)
const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`)
subscribeSshPtyNotifications({
const subscription = subscribeSshPtyNotifications({
mux: mux as never,
toAppPtyId,
dataListeners: dataListeners as never,
replayListeners: replayListeners as never,
exitListeners: exitListeners as never,
livePtyIds,
recordExit
recordExit,
providerGeneration: 7,
resolvePtyIncarnation
})
const handler = mux.onNotification.mock.calls[0]?.[0] as (
@@ -36,15 +42,33 @@ function createSubscription() {
return {
handler,
mux,
toAppPtyId,
dataListeners,
replayListeners,
exitListeners,
livePtyIds,
recordExit
recordExit,
resolvePtyIncarnation,
installReceivingActivation: subscription.installReceivingActivation
}
}
function sourceActivation(
overrides: Partial<PtySourceReceivingActivation> = {}
): PtySourceReceivingActivation {
return Object.freeze({
status: 'pending',
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
checkpointSourceEndSu: 0,
recoveryEndSu: 0,
...overrides
})
}
describe('subscribeSshPtyNotifications', () => {
it('ignores non-PTY notifications without mapping params.id', () => {
const { handler, toAppPtyId } = createSubscription()
@@ -70,6 +94,8 @@ describe('subscribeSshPtyNotifications', () => {
expect(onData).toHaveBeenCalledWith({
id: 'ssh:conn@@pty-1',
data: 'hello',
providerGeneration: 7,
ptyIncarnation: 'incarnation:pty-1',
sequenceChars: 5,
seq: 9
})
@@ -92,10 +118,689 @@ describe('subscribeSshPtyNotifications', () => {
expect(onExit).toHaveBeenCalledWith({
id: 'ssh:conn@@pty-1',
code: 0,
providerGeneration: 7,
ptyIncarnation: 'incarnation:pty-1',
incarnationId: 'incarnation-1'
})
})
it('derives exact immutable source ranges and cancels malformed frames without side effects', () => {
const {
handler,
mux,
dataListeners,
livePtyIds,
toAppPtyId,
resolvePtyIncarnation,
installReceivingActivation
} = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
livePtyIds.add('ssh:conn@@unrelated')
installReceivingActivation(
'pty-1',
sourceActivation({ checkpointSourceEndSu: 10, recoveryEndSu: 14 })
).commit()
handler('pty.data', {
id: 'pty-1',
data: 'data',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 14,
sourceLengthSu: 4
})
const acceptedSource = onData.mock.calls[0]?.[0].source
expect(Object.isFrozen(acceptedSource)).toBe(true)
const liveBeforeMalformed = new Set(livePtyIds)
toAppPtyId.mockClear()
resolvePtyIncarnation.mockClear()
handler('pty.data', {
id: 'pty-1',
data: 'bad',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 17,
sourceLengthSu: 4
})
expect(onData.mock.calls[0]?.[0]).toMatchObject({
source: {
relayPtyId: 'pty-1',
spanId: 'token-1:10:14',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1',
sourceStartSu: 10,
sourceEndSu: 14
}
})
expect(onData).toHaveBeenCalledTimes(1)
expect(livePtyIds).toEqual(liveBeforeMalformed)
expect(toAppPtyId).not.toHaveBeenCalled()
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1'
})
})
it('keeps exact source incarnation independent without mutating legacy delivery state', () => {
const { handler, dataListeners, resolvePtyIncarnation, installReceivingActivation } =
createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
handler('pty.data', { id: 'pty-1', data: 'legacy' })
installReceivingActivation(
'pty-1',
sourceActivation({ checkpointSourceEndSu: 0, recoveryEndSu: 4 })
).commit()
handler('pty.data', {
id: 'pty-1',
data: 'data',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 4,
sourceLengthSu: 4
})
expect(onData.mock.calls.map(([payload]) => payload.ptyIncarnation)).toEqual([
'incarnation:pty-1',
'incarnation-1'
])
expect(resolvePtyIncarnation).toHaveBeenCalledTimes(1)
expect(resolvePtyIncarnation).toHaveBeenCalledWith('pty-1', undefined)
})
it('drops stale delivery generations without touching their PTY or unrelated PTYs', () => {
const {
handler,
mux,
dataListeners,
livePtyIds,
toAppPtyId,
resolvePtyIncarnation,
installReceivingActivation
} = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
livePtyIds.add('ssh:conn@@unrelated')
installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 4,
ownerGeneration: 5,
deliveryToken: 'token-new',
recoveryEndSu: 3
})
).commit()
handler('pty.data', {
id: 'pty-1',
data: 'new',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
clientGeneration: 4,
ownerGeneration: 5,
sourceEndSu: 3,
sourceLengthSu: 3
})
const liveBeforeStale = new Set(livePtyIds)
toAppPtyId.mockClear()
resolvePtyIncarnation.mockClear()
handler('pty.data', {
id: 'pty-1',
data: 'old',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 3,
ownerGeneration: 4,
sourceEndSu: 6,
sourceLengthSu: 3
})
expect(onData).toHaveBeenCalledTimes(1)
expect(livePtyIds).toEqual(liveBeforeStale)
expect(toAppPtyId).not.toHaveBeenCalled()
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-old'
})
})
it('rejects same-generation token changes and source discontinuities', () => {
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
const sourceParams = {
id: 'pty-1',
ptyIncarnation: 'incarnation-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceLengthSu: 3
}
installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit()
handler('pty.data', {
...sourceParams,
data: 'one',
deliveryToken: 'token-1',
sourceEndSu: 3
})
handler('pty.data', {
...sourceParams,
data: 'two',
deliveryToken: 'token-2',
sourceEndSu: 6
})
handler('pty.data', {
...sourceParams,
data: 'gap',
deliveryToken: 'token-1',
sourceEndSu: 9
})
handler('pty.data', {
...sourceParams,
data: 'two',
deliveryToken: 'token-1',
sourceEndSu: 6
})
expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['one', 'two'])
expect(mux.request).toHaveBeenCalledTimes(2)
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-2'
})
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1'
})
})
it('accepts a strictly newer rotation, rejects late old data, and preserves new continuity', () => {
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
const frame = (
data: string,
deliveryToken: string,
clientGeneration: number,
ownerGeneration: number,
sourceEndSu: number
) => ({
id: 'pty-1',
data,
ptyIncarnation: 'incarnation-1',
deliveryToken,
clientGeneration,
ownerGeneration,
sourceEndSu,
sourceLengthSu: data.length
})
installReceivingActivation(
'pty-1',
sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 })
).commit()
handler('pty.data', frame('old', 'token-old', 2, 3, 3))
installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-new',
checkpointSourceEndSu: 10,
recoveryEndSu: 13
})
).commit()
handler('pty.data', frame('new', 'token-new', 3, 4, 13))
handler('pty.data', frame('old', 'token-old', 2, 3, 6))
handler('pty.data', frame('next', 'token-new', 3, 4, 17))
expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['old', 'new', 'next'])
expect(mux.request).toHaveBeenCalledTimes(1)
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-old'
})
})
it.each([
['client-only advance', 3, 3, 'token-client'],
['owner-only advance', 2, 4, 'token-owner'],
['crossed generations', 3, 2, 'token-crossed'],
['replayed client generation', 1, 4, 'token-replayed'],
['reused token on newer generations', 3, 4, 'token-current']
])(
'rejects a %s without replacing the accepted continuity record',
(_case, clientGeneration, ownerGeneration, deliveryToken) => {
const {
handler,
mux,
dataListeners,
livePtyIds,
toAppPtyId,
resolvePtyIncarnation,
installReceivingActivation
} = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
const base = {
id: 'pty-1',
ptyIncarnation: 'incarnation-1',
sourceLengthSu: 3
}
installReceivingActivation(
'pty-1',
sourceActivation({ deliveryToken: 'token-current', recoveryEndSu: 3 })
).commit()
handler('pty.data', {
...base,
data: 'one',
deliveryToken: 'token-current',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3
})
const liveBeforeInvalid = new Set(livePtyIds)
toAppPtyId.mockClear()
resolvePtyIncarnation.mockClear()
handler('pty.data', {
...base,
data: 'bad',
deliveryToken,
clientGeneration,
ownerGeneration,
sourceEndSu: 6
})
handler('pty.data', {
...base,
data: 'two',
deliveryToken: 'token-current',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 6
})
expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['one', 'two'])
expect(livePtyIds).toEqual(liveBeforeInvalid)
expect(toAppPtyId).toHaveBeenCalledTimes(1)
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration,
ownerGeneration,
deliveryToken
})
}
)
it('does not cancel an incomplete malformed identity or mutate provider state', () => {
const { handler, mux, dataListeners, livePtyIds, toAppPtyId, resolvePtyIncarnation } =
createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
livePtyIds.add('ssh:conn@@unrelated')
handler('pty.data', {
id: 'pty-1',
data: 'bad',
ptyIncarnation: 'incarnation-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3,
sourceLengthSu: 3
})
expect(onData).not.toHaveBeenCalled()
expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated']))
expect(toAppPtyId).not.toHaveBeenCalled()
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
expect(mux.request).not.toHaveBeenCalled()
})
it('accepts non-empty recovery from the activation checkpoint', () => {
const { handler, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
const lease = installReceivingActivation(
'pty-1',
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 })
)
handler('pty.data', {
id: 'pty-1',
data: 'next',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 8,
sourceLengthSu: 4
})
expect(onData).not.toHaveBeenCalled()
lease.commit()
expect(onData).toHaveBeenCalledWith(
expect.objectContaining({
data: 'next',
source: expect.objectContaining({ sourceStartSu: 4, sourceEndSu: 8 })
})
)
})
it('routes held and later recovery frames only to the private sink until commit', () => {
const { handler, dataListeners, livePtyIds, installReceivingActivation } = createSubscription()
const onData = vi.fn()
const onRecoveryData = vi.fn()
dataListeners.add(onData)
const lease = installReceivingActivation(
'pty-1',
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 12 })
)
const publishSource = (data: string, sourceEndSu: number): void => {
handler('pty.data', {
id: 'pty-1',
data,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu,
sourceLengthSu: 4
})
}
publishSource('held', 8)
const recoveryLease = lease.transferToRecovery(onRecoveryData)
publishSource('next', 12)
expect(onRecoveryData.mock.calls.map(([payload]) => payload.data)).toEqual(['held', 'next'])
expect(onData).not.toHaveBeenCalled()
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
recoveryLease.commit()
expect(onData).not.toHaveBeenCalled()
publishSource('live', 16)
expect(onRecoveryData).toHaveBeenCalledTimes(2)
expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'live' }))
expect(livePtyIds).toContain('ssh:conn@@pty-1')
})
it('retires an exited private recovery when its activation commits', () => {
const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } =
createSubscription()
const onData = vi.fn()
const onRecoveryData = vi.fn()
dataListeners.add(onData)
const lease = installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 4 }))
handler('pty.data', {
id: 'pty-1',
data: 'held',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 4,
sourceLengthSu: 4
})
const recoveryLease = lease.transferToRecovery(onRecoveryData)
handler('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' })
recoveryLease.commit()
handler('pty.data', {
id: 'pty-1',
data: 'late',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 8,
sourceLengthSu: 4
})
expect(onRecoveryData).toHaveBeenCalledOnce()
expect(onData).not.toHaveBeenCalled()
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
expect(mux.request).toHaveBeenCalledWith(
'pty.cancelDelivery',
expect.objectContaining({ id: 'pty-1', deliveryToken: 'token-1' })
)
})
it('retires private recovery locally and restores the exact predecessor', () => {
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
const onRecoveryData = vi.fn()
dataListeners.add(onData)
installReceivingActivation(
'pty-1',
sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 })
).commit()
handler('pty.data', {
id: 'pty-1',
data: 'pre',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3,
sourceLengthSu: 3
})
const replacement = installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-new',
checkpointSourceEndSu: 3,
recoveryEndSu: 6
})
)
handler('pty.data', {
id: 'pty-1',
data: 'new',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
clientGeneration: 3,
ownerGeneration: 4,
sourceEndSu: 6,
sourceLengthSu: 3
})
replacement.transferToRecovery(onRecoveryData).retire()
handler('pty.data', {
id: 'pty-1',
data: 'old',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 6,
sourceLengthSu: 3
})
expect(onRecoveryData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' }))
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old'])
expect(mux.request).not.toHaveBeenCalled()
})
it('rejects a stale activation without disturbing current continuity', () => {
const { handler, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit()
expect(() =>
installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 1,
ownerGeneration: 4,
deliveryToken: 'token-stale',
checkpointSourceEndSu: 3,
recoveryEndSu: 3
})
)
).toThrow('ssh_source_receiving_activation_stale')
handler('pty.data', {
id: 'pty-1',
data: 'one',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3,
sourceLengthSu: 3
})
expect(onData).toHaveBeenCalledOnce()
})
it('drops provisional frames and settles cancellation before rollback completes', async () => {
const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } =
createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
const lease = installReceivingActivation(
'pty-1',
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 })
)
handler('pty.data', {
id: 'pty-1',
data: 'next',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 8,
sourceLengthSu: 4
})
await expect(lease.rollback()).resolves.toBe(true)
expect(onData).not.toHaveBeenCalled()
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1'
})
})
it('restores the exact prior cursor when a replacement rolls back after frames', async () => {
const { handler, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
installReceivingActivation('pty-1', sourceActivation({ deliveryToken: 'token-old' })).commit()
handler('pty.data', {
id: 'pty-1',
data: 'pre',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3,
sourceLengthSu: 3
})
const replacement = installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-new',
checkpointSourceEndSu: 3,
recoveryEndSu: 3
})
)
handler('pty.data', {
id: 'pty-1',
data: 'new',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
clientGeneration: 3,
ownerGeneration: 4,
sourceEndSu: 6,
sourceLengthSu: 3
})
await replacement.rollback()
handler('pty.data', {
id: 'pty-1',
data: 'old',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 6,
sourceLengthSu: 3
})
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old'])
})
it('does not let an older lease rollback replace a newer activation', async () => {
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
const onData = vi.fn()
dataListeners.add(onData)
const older = installReceivingActivation('pty-1', sourceActivation())
const newer = installReceivingActivation(
'pty-1',
sourceActivation({
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-new'
})
)
await older.rollback()
handler('pty.data', {
id: 'pty-1',
data: 'new',
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
clientGeneration: 3,
ownerGeneration: 4,
sourceEndSu: 3,
sourceLengthSu: 3
})
newer.commit()
expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' }))
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-1'
})
expect(mux.request).not.toHaveBeenCalledWith(
'pty.cancelDelivery',
expect.objectContaining({ deliveryToken: 'token-new' })
)
})
it('ignores PTY methods with missing ids', () => {
const { handler, toAppPtyId, dataListeners } = createSubscription()
const onData = vi.fn()
@@ -105,4 +810,49 @@ describe('subscribeSshPtyNotifications', () => {
expect(toAppPtyId).not.toHaveBeenCalled()
expect(onData).not.toHaveBeenCalled()
})
it('leaves recovery and cancellation control methods to their dedicated handlers', () => {
const {
handler,
mux,
toAppPtyId,
dataListeners,
replayListeners,
exitListeners,
livePtyIds,
recordExit,
resolvePtyIncarnation
} = createSubscription()
const onData = vi.fn()
const onReplay = vi.fn()
const onExit = vi.fn()
dataListeners.add(onData)
replayListeners.add(onReplay)
exitListeners.add(onExit)
livePtyIds.add('ssh:conn@@unrelated')
for (const method of [
'pty.recoveryData',
'pty.recoveryComplete',
'pty.restoreRequired',
'pty.deliveryCanceled'
]) {
handler(method, {
id: 'pty-1',
data: 'control',
deliveryToken: 'token-1',
clientGeneration: 2,
ownerGeneration: 3
})
}
expect(toAppPtyId).not.toHaveBeenCalled()
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
expect(recordExit).not.toHaveBeenCalled()
expect(onData).not.toHaveBeenCalled()
expect(onReplay).not.toHaveBeenCalled()
expect(onExit).not.toHaveBeenCalled()
expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated']))
expect(mux.request).not.toHaveBeenCalled()
})
})
@@ -1,12 +1,35 @@
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
import { isPtyIncarnationId } from '../../shared/pty-incarnation'
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
import type {
SshPtyDataCallback,
SshPtyExitCallback,
SshPtyReplayCallback
} from './ssh-pty-provider-contract'
import { parseSshPtySourceFrame } from './ssh-pty-source-frame'
import {
SshPtySourceDeliveryLedger,
type PendingSshPtySourceData
} from './ssh-pty-source-delivery-ledger'
export type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback }
export type SshPtyRecoveryActivationLease = Readonly<{
commit: () => void
retire: () => void
}>
export type SshPtyReceivingActivationLease = Readonly<{
commit: () => void
rollback: () => Promise<boolean>
transferToRecovery: (sink: SshPtyDataCallback) => SshPtyRecoveryActivationLease
}>
export type SshPtyNotificationSubscription = Readonly<{
dispose: () => void
installReceivingActivation: (
relayPtyId: string,
activation: PtySourceReceivingActivation
) => SshPtyReceivingActivationLease
}>
export function subscribeSshPtyNotifications(args: {
mux: SshChannelMultiplexer
@@ -16,8 +39,36 @@ export function subscribeSshPtyNotifications(args: {
exitListeners: Set<SshPtyExitCallback>
livePtyIds: Set<string>
recordExit: (relayPtyId: string, incarnationId: unknown) => void
}): () => void {
return args.mux.onNotification((method, params) => {
providerGeneration: number
resolvePtyIncarnation: (relayPtyId: string, incarnationId?: unknown) => string
}): SshPtyNotificationSubscription {
const toDataPayload = (pending: PendingSshPtySourceData): Parameters<SshPtyDataCallback>[0] => {
const id = args.toAppPtyId(pending.relayPtyId)
const ptyIncarnation = pending.source
? (pending.params.ptyIncarnation as string)
: args.resolvePtyIncarnation(pending.relayPtyId, pending.params.incarnationId)
return {
id,
data: pending.data,
providerGeneration: args.providerGeneration,
ptyIncarnation,
...(typeof pending.params.rawLength === 'number'
? { sequenceChars: pending.params.rawLength }
: {}),
...(pending.params.transformed === true ? { transformed: true } : {}),
...(typeof pending.params.seq === 'number' ? { seq: pending.params.seq } : {}),
...(pending.source ? { source: pending.source } : {})
}
}
const publishData = (pending: PendingSshPtySourceData): void => {
const payload = toDataPayload(pending)
args.livePtyIds.add(payload.id)
for (const listener of args.dataListeners) {
listener(payload)
}
}
const sourceDeliveries = new SshPtySourceDeliveryLedger(args.mux, publishData)
const dispose = args.mux.onNotification((method, params) => {
// Why: mux delivers every method to generic handlers; non-PTY payloads
// (workspace.changed, fs.changed, …) have no `id` and must not reach
// toAppPtyId → startsWith.
@@ -28,14 +79,18 @@ export function subscribeSshPtyNotifications(args: {
return
}
const relayPtyId = params.id
const id = args.toAppPtyId(relayPtyId)
if (method === 'pty.exit') {
const id = args.toAppPtyId(relayPtyId)
const ptyIncarnation = args.resolvePtyIncarnation(relayPtyId, params.incarnationId)
args.recordExit(relayPtyId, params.incarnationId)
args.livePtyIds.delete(id)
sourceDeliveries.recordExit(relayPtyId)
for (const listener of args.exitListeners) {
listener({
id,
code: params.code as number,
providerGeneration: args.providerGeneration,
ptyIncarnation,
...(isPtyIncarnationId(params.incarnationId)
? { incarnationId: params.incarnationId }
: {})
@@ -43,21 +98,77 @@ export function subscribeSshPtyNotifications(args: {
}
return
}
args.livePtyIds.add(id)
if (method === 'pty.replay') {
const id = args.toAppPtyId(relayPtyId)
args.livePtyIds.add(id)
for (const listener of args.replayListeners) {
listener({ id, data: params.data as string })
}
return
}
for (const listener of args.dataListeners) {
listener({
id,
data: params.data as string,
...(typeof params.rawLength === 'number' ? { sequenceChars: params.rawLength } : {}),
...(params.transformed === true ? { transformed: true } : {}),
...(typeof params.seq === 'number' ? { seq: params.seq } : {})
const data = typeof params.data === 'string' ? params.data : ''
const sourceFrame = parseSshPtySourceFrame(params, data, relayPtyId)
if (sourceFrame.malformed) {
cancelExactSourceDelivery(args.mux, relayPtyId, params)
return
}
const pending = Object.freeze({
relayPtyId,
params,
data,
source: sourceFrame.source
})
if (sourceFrame.source) {
if (!sourceDeliveries.admit({ ...pending, source: sourceFrame.source })) {
cancelExactSourceDelivery(args.mux, relayPtyId, params)
}
return
}
publishData(pending)
})
return Object.freeze({
dispose,
installReceivingActivation: (relayPtyId, activation) => {
const lease = sourceDeliveries.install(relayPtyId, activation)
return Object.freeze({
commit: lease.commit,
rollback: lease.rollback,
transferToRecovery: (sink: SshPtyDataCallback) =>
lease.transferToRecovery((pending) => sink(toDataPayload(pending)))
})
}
})
}
function cancelExactSourceDelivery(
mux: SshChannelMultiplexer,
relayPtyId: string,
params: {
deliveryToken?: unknown
clientGeneration?: unknown
ownerGeneration?: unknown
}
): void {
if (
typeof params.deliveryToken !== 'string' ||
params.deliveryToken.length === 0 ||
!positiveSafeInteger(params.clientGeneration) ||
!positiveSafeInteger(params.ownerGeneration)
) {
return
}
try {
void mux
.request('pty.cancelDelivery', {
id: relayPtyId,
clientGeneration: params.clientGeneration,
ownerGeneration: params.ownerGeneration,
deliveryToken: params.deliveryToken
})
.catch(() => {})
} catch {}
}
function positiveSafeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) > 0
}
@@ -1,7 +1,76 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION } from '../../shared/agent-session-host-authority'
import {
AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION,
AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION
} from '../../shared/agent-session-host-authority'
import { SshChannelMultiplexer, type MultiplexerTransport } from '../ssh/ssh-channel-multiplexer'
import { encodeFrame, HEADER_LENGTH, MessageType } from '../ssh/relay-protocol'
import { SshPtyProvider } from './ssh-pty-provider'
function createTransport(): MultiplexerTransport & {
deliver: (data: Buffer) => void
written: Buffer[]
} {
let deliver = (_data: Buffer): void => {}
const written: Buffer[] = []
return {
write: (data) => {
written.push(data)
},
onData: (callback) => {
deliver = callback
},
onClose: () => {},
deliver: (data) => deliver(data),
written
}
}
function rpcFrame(payload: Record<string, unknown>, sequence: number): Buffer {
return encodeFrame(MessageType.Regular, sequence, 0, Buffer.from(JSON.stringify(payload)))
}
function responseFrame(id: number, result: unknown, sequence: number): Buffer {
return rpcFrame({ jsonrpc: '2.0', id, result }, sequence)
}
function notificationFrame(
method: string,
params: Record<string, unknown>,
sequence: number
): Buffer {
return rpcFrame({ jsonrpc: '2.0', method, params }, sequence)
}
function requestPayloads(transport: ReturnType<typeof createTransport>): Record<string, unknown>[] {
return transport.written.flatMap((frame) => {
if (frame[0] !== MessageType.Regular) {
return []
}
const payloadLength = frame.readUInt32BE(9)
return [
JSON.parse(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + payloadLength).toString()) as Record<
string,
unknown
>
]
})
}
async function waitForRequest(
transport: ReturnType<typeof createTransport>,
method: string
): Promise<Record<string, unknown>> {
for (let turn = 0; turn < 10; turn += 1) {
const request = requestPayloads(transport).find((payload) => payload.method === method)
if (request) {
return request
}
await Promise.resolve()
}
throw new Error(`request not dispatched: ${method}`)
}
describe('SSH fresh agent-session create operations', () => {
const request = vi.fn()
let provider: SshPtyProvider
@@ -35,14 +104,19 @@ describe('SSH fresh agent-session create operations', () => {
signal: undefined,
timeoutMs: 5_000
})
expect(request).toHaveBeenNthCalledWith(2, 'pty.spawn', {
cols: 80,
rows: 24,
cwd: undefined,
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
command: 'codex',
agentSessionCreateOperationId: 'a'.repeat(43)
})
expect(request).toHaveBeenNthCalledWith(
2,
'pty.spawn',
{
cols: 80,
rows: 24,
cwd: undefined,
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
command: 'codex',
agentSessionCreateOperationId: 'a'.repeat(43)
},
expect.objectContaining({ beforeResolve: expect.any(Function) })
)
})
it('does not downgrade after structured dispatch reaches an old relay', async () => {
@@ -70,13 +144,18 @@ describe('SSH fresh agent-session create operations', () => {
})
).resolves.toMatchObject({ id: 'ssh:conn-1@@pty-legacy' })
expect(request).toHaveBeenNthCalledWith(1, 'pty.spawn', {
cols: 80,
rows: 24,
cwd: undefined,
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
command: 'codex'
})
expect(request).toHaveBeenNthCalledWith(
1,
'pty.spawn',
{
cols: 80,
rows: 24,
cwd: undefined,
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
command: 'codex'
},
expect.objectContaining({ beforeResolve: expect.any(Function) })
)
})
it('re-probes a negative capability after an in-place relay upgrade', async () => {
@@ -155,4 +234,163 @@ describe('SSH fresh agent-session create operations', () => {
})
expect(request).toHaveBeenCalledTimes(2)
})
it('withholds same-turn source data until claim validation and isolates rollback', async () => {
const transport = createTransport()
const mux = new SshChannelMultiplexer(transport)
const exactProvider = new SshPtyProvider('conn-1', mux)
const onData = vi.fn()
exactProvider.onData(onData)
const claim = {
digestVersion: 1 as const,
keyId: 'key',
identityDigest: 'a'.repeat(43),
worktreeScopeDigest: 'b'.repeat(43),
agent: 'codex' as const
}
const surface = {
worktreeId: 'worktree',
tabId: 'tab',
leafId: '11111111-1111-4111-8111-111111111111',
terminalHandle: 'term_claimed'
}
const spawn = exactProvider.spawn({
cols: 80,
rows: 24,
agentSessionEnsure: { claim, surface }
})
const capabilityRequest = await waitForRequest(transport, 'pty.getCapabilities')
transport.deliver(
responseFrame(
capabilityRequest.id as number,
{ agentSessionClaimVersion: AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION },
1
)
)
const spawnRequest = await waitForRequest(transport, 'pty.spawn')
const oldActivation = {
status: 'pending',
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-old',
deliveryToken: 'token-old',
checkpointSourceEndSu: 0,
recoveryEndSu: 3
}
transport.deliver(
Buffer.concat([
responseFrame(
spawnRequest.id as number,
{
id: 'pty-1',
incarnationId: 'incarnation-old',
sourceActivation: oldActivation,
agentSessionEnsure: {
disposition: 'created',
owner: {
claim: { ...claim, identityDigest: 'c'.repeat(43) },
generation: 'generation-old',
phase: 'live',
ptyId: 'pty-1',
surface
}
}
},
2
),
notificationFrame(
'pty.data',
{
id: 'pty-1',
data: 'old',
ptyIncarnation: 'incarnation-old',
deliveryToken: 'token-old',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 3,
sourceLengthSu: 3
},
3
)
])
)
expect(onData).not.toHaveBeenCalled()
expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(false)
const shutdownRequest = await waitForRequest(transport, 'pty.shutdown')
transport.deliver(responseFrame(shutdownRequest.id as number, null, 4))
const cancelRequest = await waitForRequest(transport, 'pty.cancelDelivery')
let spawnSettled = false
void spawn.then(
() => {
spawnSettled = true
},
() => {
spawnSettled = true
}
)
await Promise.resolve()
expect(spawnSettled).toBe(false)
expect(cancelRequest.params).toEqual({
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-old'
})
const replacement = exactProvider.spawn({ cols: 80, rows: 24 })
const replacementRequest = requestPayloads(transport).findLast(
(payload) => payload.method === 'pty.spawn'
)
expect(replacementRequest).toBeDefined()
transport.deliver(
Buffer.concat([
responseFrame(
replacementRequest!.id as number,
{
id: 'pty-1',
incarnationId: 'incarnation-new',
sourceActivation: {
...oldActivation,
clientGeneration: 4,
ownerGeneration: 5,
ptyIncarnation: 'incarnation-new',
deliveryToken: 'token-new'
}
},
5
),
notificationFrame(
'pty.data',
{
id: 'pty-1',
data: 'new',
ptyIncarnation: 'incarnation-new',
deliveryToken: 'token-new',
clientGeneration: 4,
ownerGeneration: 5,
sourceEndSu: 3,
sourceLengthSu: 3
},
6
)
])
)
await expect(replacement).resolves.toMatchObject({ incarnationId: 'incarnation-new' })
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['new'])
transport.deliver(
responseFrame(
cancelRequest.id as number,
{ canceled: true, sentEndSu: 3, creditedEndSu: 0 },
7
)
)
await expect(spawn).rejects.toThrow('agent_session_ownership_unknown')
expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(true)
expect(
requestPayloads(transport).filter((payload) => payload.method === 'pty.cancelDelivery')
).toHaveLength(1)
mux.dispose()
})
})
@@ -5,19 +5,40 @@ export type RemoteCliBridgeEnv = {
relayDir: string
nodePath: string
sockPath: string
credentialFile?: string
pathDelimiter?: ':' | ';'
}
export type SshPtyDataCallback = (payload: {
id: string
data: string
providerGeneration: number
ptyIncarnation: string
sequenceChars?: number
transformed?: boolean
seq?: number
source?: Readonly<{
relayPtyId: string
spanId: string
clientGeneration: number
ownerGeneration: number
deliveryToken: string
sourceStartSu: number
sourceEndSu: number
}>
sourceMalformed?: boolean
}) => void
export type SshPtyReplayCallback = (payload: { id: string; data: string }) => void
export type SshPtyExitCallback = (payload: {
id: string
code: number
providerGeneration: number
ptyIncarnation: string
incarnationId?: PtyIncarnationId
}) => void
export type SshPtyDeliveryPauseAdapter = (args: {
id: string
providerGeneration: number
paused: boolean
}) => void
@@ -1,6 +1,18 @@
import { expect, it, vi } from 'vitest'
import { SshPtyProvider } from './ssh-pty-provider'
function sourceActivation(ptyIncarnation: string) {
return {
status: 'pending' as const,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation,
deliveryToken: `token:${ptyIncarnation}`,
checkpointSourceEndSu: 0,
recoveryEndSu: 0
}
}
it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async () => {
const mux = {
request: vi.fn(),
@@ -10,17 +22,38 @@ it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async (
isDisposed: vi.fn().mockReturnValue(false)
}
const provider = new SshPtyProvider('conn-1', mux as never)
const dataListener = vi.fn()
provider.onData(dataListener)
const exitListener = vi.fn()
provider.onExit(exitListener)
mux.request.mockImplementation(async (method: string) => {
mux.request.mockImplementation(async (method: string, _params, options) => {
if (method === 'pty.spawn') {
const result = {
id: 'pty-raced',
incarnationId: 'incarnation-raced',
sourceActivation: sourceActivation('incarnation-raced')
}
options?.beforeResolve?.(result)
const notify = mux.onNotification.mock.calls[0]?.[0]
notify?.('pty.data', {
id: 'pty-raced',
data: 'data',
ptyIncarnation: 'incarnation-raced',
deliveryToken: 'token:incarnation-raced',
clientGeneration: 2,
ownerGeneration: 3,
sourceEndSu: 4,
sourceLengthSu: 4
})
notify?.('pty.exit', {
id: 'pty-raced',
code: 0,
incarnationId: 'incarnation-raced'
})
return { id: 'pty-raced', incarnationId: 'incarnation-raced' }
return result
}
if (method === 'pty.cancelDelivery') {
return { canceled: true, sentEndSu: 4, creditedEndSu: 0 }
}
return undefined
})
@@ -32,7 +65,16 @@ it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async (
expect(exitListener).toHaveBeenCalledWith({
id: 'ssh:conn-1@@pty-raced',
code: 0,
incarnationId: 'incarnation-raced'
incarnationId: 'incarnation-raced',
providerGeneration: expect.any(Number),
ptyIncarnation: 'incarnation-raced'
})
expect(dataListener).not.toHaveBeenCalled()
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-raced',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token:incarnation-raced'
})
mux.request.mockResolvedValue({ id: 'pty-next', incarnationId: 'incarnation-next' })
await expect(provider.spawn({ cols: 80, rows: 24 })).resolves.toMatchObject({
@@ -50,15 +92,20 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch',
isDisposed: vi.fn().mockReturnValue(false)
}
const provider = new SshPtyProvider('conn-1', mux as never)
mux.request.mockImplementation(async (method: string) => {
mux.request.mockImplementation(async (method: string, _params, options) => {
if (method === 'pty.attach') {
const result = {
incarnationId: 'incarnation-existing',
sourceActivation: sourceActivation('incarnation-existing')
}
options?.beforeResolve?.(result)
const notify = mux.onNotification.mock.calls[0]?.[0]
notify?.('pty.exit', {
id: 'pty-existing',
code: 0,
incarnationId: 'incarnation-existing'
})
return { incarnationId: 'incarnation-existing' }
return result
}
return undefined
})
@@ -66,6 +113,12 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch',
await expect(
provider.spawn({ cols: 80, rows: 24, sessionId: 'ssh:conn-1@@pty-existing' })
).rejects.toThrow('agent_session_exited_during_start')
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-existing',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token:incarnation-existing'
})
mux.request.mockResolvedValue({ incarnationId: 'incarnation-next' })
await expect(
@@ -76,3 +129,44 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch',
isReattach: true
})
})
it('returns a provisional source activation lease to reconnect authority', async () => {
const mux = {
request: vi.fn(),
notify: vi.fn(),
onNotification: vi.fn(),
dispose: vi.fn(),
isDisposed: vi.fn().mockReturnValue(false)
}
const provider = new SshPtyProvider('conn-1', mux as never)
const response = {
incarnationId: 'incarnation-reconnect',
sourceActivation: {
...sourceActivation('incarnation-reconnect'),
deliveryToken: 'token-reconnect',
checkpointSourceEndSu: 4,
recoveryEndSu: 8
}
}
mux.request.mockImplementation(async (method: string, _params, options) => {
if (method === 'pty.cancelDelivery') {
return { canceled: true, sentEndSu: 8, creditedEndSu: 4 }
}
if (method !== 'pty.attach') {
return undefined
}
options?.beforeResolve?.(response)
return response
})
const result = await provider.attachForReconnect('ssh:conn-1@@pty-1')
await result.sourceActivationLease?.rollback()
expect(Object.isFrozen(result.sourceActivation)).toBe(true)
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-reconnect'
})
})
@@ -0,0 +1,147 @@
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
import type {
SshPtyDataCallback,
SshPtyDeliveryPauseAdapter,
SshPtyExitCallback,
SshPtyReplayCallback
} from './ssh-pty-provider-contract'
import {
subscribeSshPtyNotifications,
type SshPtyNotificationSubscription,
type SshPtyReceivingActivationLease
} from './ssh-pty-notification-routing'
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
export class SshPtyProviderOutputState {
private readonly dataListeners = new Set<SshPtyDataCallback>()
private readonly replayListeners = new Set<SshPtyReplayCallback>()
private readonly exitListeners = new Set<SshPtyExitCallback>()
private readonly incarnationByRelayPtyId = new Map<string, string>()
private readonly pausedRelayPtyIds = new Set<string>()
private deliveryPauseAdapter: SshPtyDeliveryPauseAdapter | null = null
private legacyIncarnationSerial = 1
private subscription: SshPtyNotificationSubscription | null
constructor(
private readonly providerGeneration: number,
args: {
mux: SshChannelMultiplexer
toAppPtyId: (id: string) => string
livePtyIds: Set<string>
recordExit: (relayPtyId: string, incarnationId: unknown) => void
}
) {
this.subscription = subscribeSshPtyNotifications({
...args,
dataListeners: this.dataListeners,
replayListeners: this.replayListeners,
exitListeners: this.exitListeners,
providerGeneration,
resolvePtyIncarnation: (relayPtyId, incarnationId) =>
this.resolvePtyIncarnation(relayPtyId, incarnationId),
recordExit: (relayPtyId, incarnationId) => {
args.recordExit(relayPtyId, incarnationId)
this.incarnationByRelayPtyId.delete(relayPtyId)
this.pausedRelayPtyIds.delete(relayPtyId)
}
})
}
dispose(): void {
this.resumePausedDeliveries()
this.subscription?.dispose()
this.subscription = null
this.dataListeners.clear()
this.replayListeners.clear()
this.exitListeners.clear()
this.incarnationByRelayPtyId.clear()
this.deliveryPauseAdapter = null
}
onData(callback: SshPtyDataCallback): () => void {
this.dataListeners.add(callback)
return () => this.dataListeners.delete(callback)
}
onReplay(callback: SshPtyReplayCallback): () => void {
this.replayListeners.add(callback)
return () => this.replayListeners.delete(callback)
}
onExit(callback: SshPtyExitCallback): () => void {
this.exitListeners.add(callback)
return () => this.exitListeners.delete(callback)
}
setDeliveryPauseAdapter(adapter: SshPtyDeliveryPauseAdapter | null): void {
if (adapter !== this.deliveryPauseAdapter) {
this.resumePausedDeliveries()
}
this.deliveryPauseAdapter = adapter
}
hasDeliveryPauseAdapter(): boolean {
return this.deliveryPauseAdapter !== null
}
pause(id: string): void {
if (!this.deliveryPauseAdapter || this.pausedRelayPtyIds.has(id)) {
return
}
this.pausedRelayPtyIds.add(id)
this.deliveryPauseAdapter({ id, providerGeneration: this.providerGeneration, paused: true })
}
resume(id: string): void {
if (!this.deliveryPauseAdapter || !this.pausedRelayPtyIds.delete(id)) {
return
}
this.deliveryPauseAdapter({ id, providerGeneration: this.providerGeneration, paused: false })
}
installReceivingActivation(
relayPtyId: string,
activation: PtySourceReceivingActivation
): SshPtyReceivingActivationLease {
if (!this.subscription) {
throw new Error('ssh_source_receiving_activation_disposed')
}
return this.subscription.installReceivingActivation(relayPtyId, activation)
}
rememberPtyIncarnation(relayPtyId: string, incarnationId: unknown): void {
if (
!this.incarnationByRelayPtyId.has(relayPtyId) &&
typeof incarnationId === 'string' &&
incarnationId.length > 0
) {
this.incarnationByRelayPtyId.set(relayPtyId, incarnationId)
}
}
private resolvePtyIncarnation(relayPtyId: string, incarnationId: unknown): string {
this.rememberPtyIncarnation(relayPtyId, incarnationId)
let resolved = this.incarnationByRelayPtyId.get(relayPtyId)
if (!resolved) {
resolved = `legacy:${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}`
this.incarnationByRelayPtyId.set(relayPtyId, resolved)
}
return resolved
}
private resumePausedDeliveries(): void {
const adapter = this.deliveryPauseAdapter
if (!adapter) {
this.pausedRelayPtyIds.clear()
return
}
for (const id of this.pausedRelayPtyIds) {
try {
adapter({ id, providerGeneration: this.providerGeneration, paused: false })
} catch {
/* Generation close is the fallback cleanup proof. */
}
}
this.pausedRelayPtyIds.clear()
}
}
@@ -13,7 +13,7 @@ function createMockMux(): MockMultiplexer {
return {
request: vi.fn().mockResolvedValue(undefined),
notify: vi.fn(),
onNotification: vi.fn(),
onNotification: vi.fn().mockReturnValue(vi.fn()),
dispose: vi.fn(),
isDisposed: vi.fn().mockReturnValue(false)
}
@@ -135,10 +135,17 @@ describe('SshPtyProvider process listings and events', () => {
notify('pty.replay', { id: 'pty-1', data: 'buffered output' })
notify('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' })
expect(dataHandler).toHaveBeenNthCalledWith(1, { id: scopedPty1, data: 'output' })
expect(dataHandler).toHaveBeenNthCalledWith(1, {
id: scopedPty1,
data: 'output',
providerGeneration: 1,
ptyIncarnation: 'legacy:1:1:pty-1'
})
expect(dataHandler).toHaveBeenNthCalledWith(2, {
id: scopedPty1,
data: '',
providerGeneration: 1,
ptyIncarnation: 'legacy:1:1:pty-1',
sequenceChars: 9,
seq: 9,
transformed: true
@@ -147,10 +154,28 @@ describe('SshPtyProvider process listings and events', () => {
expect(exitHandler).toHaveBeenCalledWith({
id: scopedPty1,
code: 0,
providerGeneration: 1,
ptyIncarnation: 'legacy:1:1:pty-1',
incarnationId: 'incarnation-1'
})
})
it('keeps fallback admission identity stable when spawn metadata arrives later', async () => {
const dataHandler = vi.fn()
provider.onData(dataHandler)
const notify = mux.onNotification.mock.calls[0][0]
notify('pty.data', { id: 'pty-1', data: 'before-response' })
mux.request.mockResolvedValue({ id: 'pty-1', incarnationId: 'incarnation-1' })
await provider.spawn({ cols: 80, rows: 24 })
notify('pty.data', { id: 'pty-1', data: 'after-response' })
expect(dataHandler.mock.calls.map(([payload]) => payload.ptyIncarnation)).toEqual([
'legacy:1:1:pty-1',
'legacy:1:1:pty-1'
])
})
it('supports listener removal, fanout, and connection namespaces', () => {
const removed = vi.fn()
const first = vi.fn()
@@ -170,6 +195,29 @@ describe('SshPtyProvider process listings and events', () => {
const other = vi.fn()
otherProvider.onData(other)
otherMux.onNotification.mock.calls[0][0]('pty.data', { id: 'pty-1', data: 'second' })
expect(other).toHaveBeenCalledWith({ id: 'ssh:conn-2@@pty-1', data: 'second' })
expect(other).toHaveBeenCalledWith({
id: 'ssh:conn-2@@pty-1',
data: 'second',
providerGeneration: 1,
ptyIncarnation: 'legacy:1:1:pty-1'
})
})
it('scopes pause delivery adapters and resumes them during cleanup', () => {
const adapter = vi.fn()
provider.setPtyDeliveryPauseAdapter(adapter)
provider.pauseProducer(scopedPty1)
provider.pauseProducer(scopedPty1)
provider.resumeProducer(scopedPty1)
provider.pauseProducer(scopedPty1)
provider.dispose()
expect(adapter.mock.calls).toEqual([
[{ id: 'pty-1', providerGeneration: 1, paused: true }],
[{ id: 'pty-1', providerGeneration: 1, paused: false }],
[{ id: 'pty-1', providerGeneration: 1, paused: true }],
[{ id: 'pty-1', providerGeneration: 1, paused: false }]
])
})
})
@@ -0,0 +1,51 @@
import { describe, expect, it, vi } from 'vitest'
import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors'
import { SshPtyProvider } from './ssh-pty-provider'
describe('SSH PTY provider session reattach incarnation', () => {
it('remembers the authoritative incarnation before a legacy exit arrives', async () => {
let notify: ((method: string, params: Record<string, unknown>) => void) | undefined
const mux = {
request: vi.fn().mockResolvedValue({ incarnationId: 'incarnation-reattached' }),
notify: vi.fn(),
onNotification: vi.fn(
(callback: (method: string, params: Record<string, unknown>) => void) => {
notify = callback
return vi.fn()
}
)
}
const provider = new SshPtyProvider('conn-1', mux as never)
const onExit = vi.fn()
provider.onExit(onExit)
await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })
notify?.('pty.exit', { id: 'pty-old', code: 0 })
expect(onExit).toHaveBeenCalledWith(
expect.objectContaining({
id: 'ssh:conn-1@@pty-old',
ptyIncarnation: 'incarnation-reattached'
})
)
})
it('fails closed when generic reattach requires source restoration', async () => {
const mux = {
request: vi.fn().mockResolvedValue({
incarnationId: 'incarnation-reattached',
sourceRecovery: {
status: 'restoreRequired',
reason: 'checkpointUnavailable'
}
}),
notify: vi.fn(),
onNotification: vi.fn().mockReturnValue(vi.fn())
}
const provider = new SshPtyProvider('conn-1', mux as never)
await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow(
`${SSH_SESSION_EXPIRED_ERROR}: pty-old`
)
})
})
+91 -51
View File
@@ -22,6 +22,14 @@ function createMockMux(): MockMultiplexer {
}
}
const sourceActivationRequestOptions = expect.objectContaining({
beforeResolve: expect.any(Function)
})
function expectRequest(request: ReturnType<typeof vi.fn>, ...expected: unknown[]): void {
expect(request.mock.calls.map((call) => call.slice(0, expected.length))).toContainEqual(expected)
}
describe('SshPtyProvider', () => {
let mux: MockMultiplexer
let provider: SshPtyProvider
@@ -184,7 +192,7 @@ describe('SshPtyProvider', () => {
await expect(
provider.spawn({ cols: 80, rows: 24, agentSessionEnsure: { claim, surface } })
).rejects.toThrow('agent_session_ownership_unknown')
expect(mux.request).toHaveBeenCalledWith('pty.shutdown', {
expectRequest(mux.request, 'pty.shutdown', {
id: 'pty-malformed',
immediate: true
})
@@ -261,7 +269,7 @@ describe('SshPtyProvider', () => {
const result = await provider.spawn({ cols: 80, rows: 24 })
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 80,
rows: 24,
cwd: undefined,
@@ -289,7 +297,8 @@ describe('SshPtyProvider', () => {
await provider.spawn({ cols: 80, rows: 24, startupIngress })
expect(mux.request).toHaveBeenCalledWith(
expectRequest(
mux.request,
'pty.spawn',
expect.objectContaining({
startupIngressVersion: PTY_STARTUP_INGRESS_VERSION,
@@ -308,7 +317,7 @@ describe('SshPtyProvider', () => {
env: { FOO: 'bar' }
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: '/home/user',
@@ -326,7 +335,8 @@ describe('SshPtyProvider', () => {
launchAgent: 'claude'
})
expect(mux.request).toHaveBeenCalledWith(
expectRequest(
mux.request,
'pty.spawn',
expect.objectContaining({
command: 'cd /repo && custom-agent-wrapper',
@@ -345,7 +355,7 @@ describe('SshPtyProvider', () => {
tabId: 'tab-a'
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -365,7 +375,7 @@ describe('SshPtyProvider', () => {
terminalWindowsWslDistro: 'Ubuntu'
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -384,7 +394,7 @@ describe('SshPtyProvider', () => {
env: { [POWERLEVEL10K_WIZARD_DISABLE_ENV]: 'already-set' }
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -402,7 +412,7 @@ describe('SshPtyProvider', () => {
envToDelete: [POWERLEVEL10K_WIZARD_DISABLE_ENV]
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -426,7 +436,7 @@ describe('SshPtyProvider', () => {
envToDelete
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -452,7 +462,7 @@ describe('SshPtyProvider', () => {
startupCommandDelivery: 'shell-ready'
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -478,7 +488,7 @@ describe('SshPtyProvider', () => {
env: { PATH: '/usr/bin', ORCA_TERMINAL_HANDLE: 'term_ssh' }
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -509,7 +519,7 @@ describe('SshPtyProvider', () => {
env: { ORCA_TERMINAL_HANDLE: 'term_ssh' }
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -540,7 +550,7 @@ describe('SshPtyProvider', () => {
env: { Path: 'C:/Windows/System32;C:/Tools' }
})
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
expectRequest(mux.request, 'pty.spawn', {
cols: 120,
rows: 40,
cwd: undefined,
@@ -563,12 +573,17 @@ describe('SshPtyProvider', () => {
const result = await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
id: 'pty-old',
cols: 80,
rows: 24,
suppressReplayNotification: true
})
expectRequest(
mux.request,
'pty.attach',
{
id: 'pty-old',
cols: 80,
rows: 24,
suppressReplayNotification: true
},
sourceActivationRequestOptions
)
expect(result).toEqual({
id: 'ssh:conn-1@@pty-old',
isReattach: true,
@@ -590,7 +605,8 @@ describe('SshPtyProvider', () => {
}
})
expect(mux.request).toHaveBeenCalledWith(
expectRequest(
mux.request,
'pty.attach',
expect.not.objectContaining({ startupIngress: expect.anything() })
)
@@ -606,7 +622,7 @@ describe('SshPtyProvider', () => {
sessionId: 'ssh:conn-1@@pty-old'
})
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
expectRequest(mux.request, 'pty.attach', {
id: 'pty-old',
cols: 80,
rows: 24,
@@ -630,7 +646,7 @@ describe('SshPtyProvider', () => {
tabId: 'tab-a'
})
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
expectRequest(mux.request, 'pty.attach', {
id: 'pty-old',
cols: 80,
rows: 24,
@@ -647,12 +663,17 @@ describe('SshPtyProvider', () => {
'SSH_SESSION_EXPIRED: pty-old'
)
expect(mux.request).toHaveBeenNthCalledWith(1, 'pty.attach', {
id: 'pty-old',
cols: 80,
rows: 24,
suppressReplayNotification: true
})
expect(mux.request).toHaveBeenNthCalledWith(
1,
'pty.attach',
{
id: 'pty-old',
cols: 80,
rows: 24,
suppressReplayNotification: true
},
sourceActivationRequestOptions
)
expect(mux.request).toHaveBeenCalledTimes(1)
})
@@ -669,7 +690,7 @@ describe('SshPtyProvider', () => {
it('attach sends pty.attach request', async () => {
await provider.attach(scopedPty1)
expect(mux.request).toHaveBeenCalledWith('pty.attach', { id: 'pty-1' })
expectRequest(mux.request, 'pty.attach', { id: 'pty-1' })
})
it('attachForReconnect returns replay without relay notification', async () => {
@@ -684,10 +705,18 @@ describe('SshPtyProvider', () => {
replay: 'restored output',
incarnationId: 'incarnation-reconnect'
})
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
id: 'pty-1',
suppressReplayNotification: true
})
expectRequest(
mux.request,
'pty.attach',
{
id: 'pty-1',
suppressReplayNotification: true
},
expect.objectContaining({
timeoutMs: 10_000,
beforeResolve: expect.any(Function)
})
)
})
it('keeps missing incarnation compatible with an old relay', async () => {
@@ -712,12 +741,20 @@ describe('SshPtyProvider', () => {
tabId: 'tab-a'
})
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
id: 'pty-1',
suppressReplayNotification: true,
expectedPaneKey: 'tab-a:leaf-a',
expectedTabId: 'tab-a'
})
expectRequest(
mux.request,
'pty.attach',
{
id: 'pty-1',
suppressReplayNotification: true,
expectedPaneKey: 'tab-a:leaf-a',
expectedTabId: 'tab-a'
},
expect.objectContaining({
timeoutMs: 10_000,
beforeResolve: expect.any(Function)
})
)
})
it('write sends pty.data notification', () => {
@@ -734,7 +771,7 @@ describe('SshPtyProvider', () => {
mux.request.mockResolvedValue({ cols: 120, rows: 40 })
await expect(provider.getAppliedSize(scopedPty1)).resolves.toEqual({ cols: 120, rows: 40 })
expect(mux.request).toHaveBeenCalledWith('pty.getSize', { id: 'pty-1' }, { timeoutMs: 1_000 })
expectRequest(mux.request, 'pty.getSize', { id: 'pty-1' }, { timeoutMs: 1_000 })
})
it('caches only an old relay method-not-found response', async () => {
@@ -759,7 +796,8 @@ describe('SshPtyProvider', () => {
it('shutdown sends pty.shutdown request', async () => {
await provider.shutdown(scopedPty1, { immediate: true })
expect(mux.request).toHaveBeenCalledWith(
expectRequest(
mux.request,
'pty.shutdown',
{
id: 'pty-1',
@@ -772,7 +810,8 @@ describe('SshPtyProvider', () => {
it('shutdown forwards keepHistory: true over the relay', async () => {
await provider.shutdown(scopedPty1, { immediate: true, keepHistory: true })
expect(mux.request).toHaveBeenCalledWith(
expectRequest(
mux.request,
'pty.shutdown',
{
id: 'pty-1',
@@ -789,7 +828,8 @@ describe('SshPtyProvider', () => {
vi.useFakeTimers()
try {
await provider.shutdown(scopedPty1, { immediate: true, deadlineMs: Date.now() + 4321 })
expect(mux.request).toHaveBeenCalledWith(
expectRequest(
mux.request,
'pty.shutdown',
{ id: 'pty-1', immediate: true, keepHistory: false },
{ timeoutMs: 4321 }
@@ -801,19 +841,19 @@ describe('SshPtyProvider', () => {
it('sendSignal sends pty.sendSignal request', async () => {
await provider.sendSignal(scopedPty1, 'SIGINT')
expect(mux.request).toHaveBeenCalledWith('pty.sendSignal', { id: 'pty-1', signal: 'SIGINT' })
expectRequest(mux.request, 'pty.sendSignal', { id: 'pty-1', signal: 'SIGINT' })
})
it('getCwd sends pty.getCwd request', async () => {
mux.request.mockResolvedValue('/home/user/project')
const cwd = await provider.getCwd(scopedPty1)
expect(cwd).toBe('/home/user/project')
expect(mux.request).toHaveBeenCalledWith('pty.getCwd', { id: 'pty-1' })
expectRequest(mux.request, 'pty.getCwd', { id: 'pty-1' })
})
it('clearBuffer sends pty.clearBuffer request', async () => {
await provider.clearBuffer(scopedPty1)
expect(mux.request).toHaveBeenCalledWith('pty.clearBuffer', { id: 'pty-1' })
expectRequest(mux.request, 'pty.clearBuffer', { id: 'pty-1' })
})
it('acknowledgeDataEvent sends pty.ackData notification', () => {
@@ -825,14 +865,14 @@ describe('SshPtyProvider', () => {
mux.request.mockResolvedValue(true)
const result = await provider.hasChildProcesses(scopedPty1)
expect(result).toBe(true)
expect(mux.request).toHaveBeenCalledWith('pty.hasChildProcesses', { id: 'pty-1' })
expectRequest(mux.request, 'pty.hasChildProcesses', { id: 'pty-1' })
})
it('getForegroundProcess returns process name', async () => {
mux.request.mockResolvedValue('node')
const result = await provider.getForegroundProcess(scopedPty1)
expect(result).toBe('node')
expect(mux.request).toHaveBeenCalledWith('pty.getForegroundProcess', { id: 'pty-1' })
expectRequest(mux.request, 'pty.getForegroundProcess', { id: 'pty-1' })
})
it('preserves unavailable process inspection', async () => {
@@ -844,7 +884,7 @@ describe('SshPtyProvider', () => {
mux.request.mockResolvedValue(inspection)
await expect(provider.inspectProcess(scopedPty1)).resolves.toEqual(inspection)
expect(mux.request).toHaveBeenCalledWith('pty.inspectProcess', { id: 'pty-1' })
expectRequest(mux.request, 'pty.inspectProcess', { id: 'pty-1' })
})
it('serializes scoped app ids using raw relay ids', async () => {
@@ -853,7 +893,7 @@ describe('SshPtyProvider', () => {
const result = await provider.serialize([scopedPty1])
expect(result).toBe('serialized')
expect(mux.request).toHaveBeenCalledWith('pty.serialize', { ids: ['pty-1'] })
expectRequest(mux.request, 'pty.serialize', { ids: ['pty-1'] })
})
it('rejects scoped ids owned by another SSH connection', async () => {
+113 -118
View File
@@ -5,25 +5,24 @@ import { createSshPtyAppliedSizeReader } from './ssh-pty-applied-size'
import type {
RemoteCliBridgeEnv,
SshPtyDataCallback,
SshPtyDeliveryPauseAdapter,
SshPtyExitCallback,
SshPtyReplayCallback
} from './ssh-pty-provider-contract'
import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing'
import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation'
import {
assertSshAgentSessionCreateResult,
requestSshAgentSessionCreate
} from './ssh-agent-session-create-operation'
import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state'
import { spawnFreshSshPty } from './ssh-agent-session-create-operation'
import { mapSshPtyProcessList } from './ssh-agent-session-process-list'
import {
parseSshPtyAttachResult,
requestSshPtyAttach,
reattachSshPtySessionWithExitFence,
type PtySourceRecoveryRequest,
type SshPtyAttachResult
} from './ssh-pty-session-reattach'
import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request'
import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities'
import type { PtyProcessInspection } from './pty-process-inspection'
import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors'
// Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch.
function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined {
@@ -34,58 +33,43 @@ function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: numbe
export class SshPtyProvider implements IPtyProvider {
private mux: SshChannelMultiplexer
private connectionId: string
private dataListeners = new Set<SshPtyDataCallback>()
private replayListeners = new Set<SshPtyReplayCallback>()
private exitListeners = new Set<SshPtyExitCallback>()
private livePtyIds = new Set<string>()
// Why: stale notification callbacks must not outlive a disconnected provider.
private unsubscribeNotifications: (() => void) | null = null
readonly getAppliedSize: NonNullable<IPtyProvider['getAppliedSize']>
private readonly agentSessionCapabilities: SshAgentSessionCapabilities
private spawnExitRaces = new SshPtySpawnExitRaceTracker()
private readonly outputState: SshPtyProviderOutputState
constructor(
connectionId: string,
mux: SshChannelMultiplexer,
private readonly remoteCliBridgeEnv?: RemoteCliBridgeEnv
private readonly remoteCliBridgeEnv?: RemoteCliBridgeEnv,
readonly providerGeneration = 1
) {
this.connectionId = connectionId
this.mux = mux
this.agentSessionCapabilities = new SshAgentSessionCapabilities(mux)
this.getAppliedSize = createSshPtyAppliedSizeReader(mux, connectionId)
this.unsubscribeNotifications = subscribeSshPtyNotifications({
this.outputState = new SshPtyProviderOutputState(providerGeneration, {
mux,
toAppPtyId: (id) => this.toAppPtyId(id),
dataListeners: this.dataListeners,
replayListeners: this.replayListeners,
exitListeners: this.exitListeners,
livePtyIds: this.livePtyIds,
recordExit: (relayPtyId, incarnationId) =>
recordExit: (relayPtyId, incarnationId) => {
this.spawnExitRaces.recordExit(relayPtyId, incarnationId)
}
})
}
dispose(): void {
if (this.unsubscribeNotifications) {
this.unsubscribeNotifications()
this.unsubscribeNotifications = null
}
this.dataListeners.clear()
this.replayListeners.clear()
this.exitListeners.clear()
this.outputState.dispose()
this.livePtyIds.clear()
}
getConnectionId = (): string => this.connectionId
private toRelayPtyId(id: string): string {
return toRelaySshPtyId(this.connectionId, id)
}
private toRelayPtyId = (id: string): string => toRelaySshPtyId(this.connectionId, id)
private toAppPtyId(id: string): string {
return toAppSshPtyId(this.connectionId, id)
}
private toAppPtyId = (id: string): string => toAppSshPtyId(this.connectionId, id)
async spawn(opts: PtySpawnOptions): Promise<PtySpawnResult> {
if (opts.agentSessionEnsure && opts.sessionId) {
@@ -101,15 +85,36 @@ export class SshPtyProvider implements IPtyProvider {
}
}
if (opts.sessionId) {
const result = await reattachSshPtySessionWithExitFence({
mux: this.mux,
connectionId: this.connectionId,
sessionId: opts.sessionId,
options: opts,
exitRaceTracker: this.spawnExitRaces
})
this.livePtyIds.add(result.id)
return result
let result: Awaited<ReturnType<typeof reattachSshPtySessionWithExitFence>> | undefined
try {
result = await reattachSshPtySessionWithExitFence({
mux: this.mux,
connectionId: this.connectionId,
sessionId: opts.sessionId,
options: opts,
exitRaceTracker: this.spawnExitRaces,
installSourceActivation: (relayPtyId, activation) =>
this.outputState.installReceivingActivation(relayPtyId, activation),
rememberPtyIncarnation: (relayPtyId, incarnationId) =>
this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId)
})
if (result.sourceRecovery?.status === 'restoreRequired') {
throw new Error(
`${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}`
)
}
this.livePtyIds.add(result.id)
result.sourceActivationLease?.commit()
const {
sourceActivationLease: _lease,
sourceRecovery: _sourceRecovery,
...spawnResult
} = result
return spawnResult
} catch (error) {
result?.sourceActivationLease?.rollback()
throw error
}
}
const supportsCreateOperation = opts.agentSessionCreateOperationId
@@ -122,65 +127,22 @@ export class SshPtyProvider implements IPtyProvider {
// Why: host routing owns legacy selection; a changed relay must not downgrade after dispatch.
throw new Error('execution_owner_unavailable')
}
const operation = this.spawnExitRaces.begin()
try {
const result = await requestSshAgentSessionCreate({
mux: this.mux,
operationId: opts.agentSessionCreateOperationId,
signal: opts.signal,
params: buildSshPtySpawnRequest({
options: opts,
remoteCliBridgeEnv: this.remoteCliBridgeEnv,
supportsCreateOperation
})
})
if (opts.agentSessionCreateOperationId) {
assertSshAgentSessionCreateResult(result)
}
const spawnResult = result as PtySpawnResult
if (this.spawnExitRaces.didMatchingExitArrive(operation, spawnResult)) {
// Why: relay notification can share the response batch; no controller registration may follow.
throw Object.assign(new Error('agent_session_exited_during_start'), {
agentSessionOperationOutcome: 'unknown' as const
})
}
const claimed = spawnResult.agentSessionEnsure
if (opts.agentSessionEnsure) {
const validation = validateClaimedSshSpawn(spawnResult, opts.agentSessionEnsure)
if (!validation.valid) {
if (validation.cleanup === 'created' && typeof spawnResult.id === 'string') {
try {
// Why: immediate relay shutdown resolves only after physical exit;
// a best-effort graceful request cannot prove the duplicate is gone.
await this.mux.request('pty.shutdown', { id: spawnResult.id, immediate: true })
} catch {
throw new Error('execution_owner_unavailable')
}
}
throw new Error(validation.error)
}
}
const id = this.toAppPtyId(spawnResult.id)
this.livePtyIds.add(id)
return {
...spawnResult,
id,
...(claimed
? {
agentSessionEnsure: {
...claimed,
owner: {
...claimed.owner,
ptyId: this.toAppPtyId(claimed.owner.ptyId)
}
}
}
: {}),
...(opts.sessionId ? { sessionExpired: true } : {})
}
} finally {
this.spawnExitRaces.finish(operation)
}
return await spawnFreshSshPty({
mux: this.mux,
options: opts,
params: buildSshPtySpawnRequest({
options: opts,
remoteCliBridgeEnv: this.remoteCliBridgeEnv,
supportsCreateOperation
}),
exitRaceTracker: this.spawnExitRaces,
installSourceActivation: (id, activation) =>
this.outputState.installReceivingActivation(id, activation),
rememberPtyIncarnation: (id, incarnation) =>
this.outputState.rememberPtyIncarnation(id, incarnation),
acceptLivePty: (id) => this.livePtyIds.add(id),
toAppPtyId: this.toAppPtyId
})
}
async supportsAgentSessionClaims(options: { signal?: AbortSignal } = {}): Promise<boolean> {
@@ -198,25 +160,46 @@ export class SshPtyProvider implements IPtyProvider {
}
async attach(id: string): Promise<void> {
await this.mux.request('pty.attach', { id: this.toRelayPtyId(id) })
const relayPtyId = this.toRelayPtyId(id)
await requestSshPtyAttach({
mux: this.mux,
relayPtyId,
params: { id: relayPtyId },
commitSourceActivation: true,
installSourceActivation: (ptyId, activation) =>
this.outputState.installReceivingActivation(ptyId, activation),
rememberPtyIncarnation: (ptyId, incarnationId) =>
this.outputState.rememberPtyIncarnation(ptyId, incarnationId)
})
}
async attachForReconnect(
id: string,
expected?: { paneKey?: string; tabId?: string }
expected?: { paneKey?: string; tabId?: string },
sourceRecovery?: PtySourceRecoveryRequest
): Promise<SshPtyAttachResult> {
// Why: reconnect owns replay delivery so stale/duplicate attach results can
// be filtered before they reach the renderer. The expected identity lets the
// relay reject a cross-generation id collision instead of reattaching this
// lease to a different pane's freshly spawned PTY.
return parseSshPtyAttachResult(
await this.mux.request('pty.attach', {
id: this.toRelayPtyId(id),
suppressReplayNotification: true,
...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}),
...(expected?.tabId ? { expectedTabId: expected.tabId } : {})
})
)
const params = {
id: this.toRelayPtyId(id),
suppressReplayNotification: true,
...(sourceRecovery ? { sourceRecovery } : {}),
...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}),
...(expected?.tabId ? { expectedTabId: expected.tabId } : {})
}
const relayPtyId = this.toRelayPtyId(id)
return await requestSshPtyAttach({
mux: this.mux,
relayPtyId,
params,
timeoutMs: 10_000,
installSourceActivation: (ptyId, activation) =>
this.outputState.installReceivingActivation(ptyId, activation),
rememberPtyIncarnation: (ptyId, incarnationId) =>
this.outputState.rememberPtyIncarnation(ptyId, incarnationId)
})
}
write(id: string, data: string): void {
@@ -308,6 +291,8 @@ export class SshPtyProvider implements IPtyProvider {
const processes = mapSshPtyProcessList(result as PtyProcessInfo[], (id) => this.toAppPtyId(id))
for (const process of processes) {
this.livePtyIds.add(process.id)
const relayPtyId = this.toRelayPtyId(process.id)
this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId)
}
return processes
}
@@ -326,18 +311,28 @@ export class SshPtyProvider implements IPtyProvider {
return result as { name: string; path: string }[]
}
onData(callback: SshPtyDataCallback): () => void {
this.dataListeners.add(callback)
return () => this.dataListeners.delete(callback)
onData = (callback: SshPtyDataCallback): (() => void) => this.outputState.onData(callback)
onReplay = (callback: SshPtyReplayCallback): (() => void) => this.outputState.onReplay(callback)
onExit = (callback: SshPtyExitCallback): (() => void) => this.outputState.onExit(callback)
setPtyDeliveryPauseAdapter(adapter: SshPtyDeliveryPauseAdapter | null): void {
this.outputState.setDeliveryPauseAdapter(adapter)
}
onReplay(callback: SshPtyReplayCallback): () => void {
this.replayListeners.add(callback)
return () => this.replayListeners.delete(callback)
hasPtyDeliveryPauseAdapter(): boolean {
return this.outputState.hasDeliveryPauseAdapter()
}
onExit(callback: SshPtyExitCallback): () => void {
this.exitListeners.add(callback)
return () => this.exitListeners.delete(callback)
pauseProducer(id: string): void {
this.outputState.pause(this.toRelayPtyId(id))
}
resumeProducer(id: string): void {
this.outputState.resume(this.toRelayPtyId(id))
}
closeOutputIntake(reason: string): void {
this.mux.dispose('connection_lost')
console.error('[ssh-pty-provider] closed after bounded output intake failure', { reason })
}
}
+163 -10
View File
@@ -9,10 +9,27 @@ import {
import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id'
import type { PtySpawnOptions, PtySpawnResult } from './types'
import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
import type {
PtySourceRecoveryRequest,
PtySourceRecoveryResult
} from '../../shared/pty-source-recovery-contract'
import {
parsePtySourceReceivingActivation,
type PtySourceReceivingActivation
} from '../../shared/pty-source-receiving-activation'
import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing'
export type SshPtyAttachResult = {
replay?: string
incarnationId?: PtyIncarnationId
sourceRecovery?: PtySourceRecoveryResult
sourceActivation?: PtySourceReceivingActivation
sourceActivationLease?: SshPtyReceivingActivationLease
}
type SshPtyReattachResult = PtySpawnResult & {
sourceRecovery?: PtySourceRecoveryResult
sourceActivationLease?: SshPtyReceivingActivationLease
}
export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult {
@@ -22,7 +39,12 @@ export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult {
if (typeof value !== 'object' || Array.isArray(value)) {
throw new Error('Invalid SSH PTY attach response')
}
const result = value as { replay?: unknown; incarnationId?: unknown }
const result = value as {
replay?: unknown
incarnationId?: unknown
sourceRecovery?: unknown
sourceActivation?: unknown
}
if (result.replay !== undefined && typeof result.replay !== 'string') {
throw new Error('Invalid SSH PTY attach replay')
}
@@ -30,34 +52,156 @@ export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult {
// Why: a present-but-invalid identity cannot safely fence delayed exits from a reused relay id.
throw new Error('Invalid SSH PTY attach incarnation')
}
const sourceRecovery = parseSourceRecoveryResult(result.sourceRecovery)
const sourceActivation = parsePtySourceReceivingActivation(result.sourceActivation)
const activation =
sourceActivation ?? (sourceRecovery?.status === 'pending' ? sourceRecovery : undefined)
if (
activation &&
(!isPtyIncarnationId(result.incarnationId) ||
activation.ptyIncarnation !== result.incarnationId ||
(sourceRecovery?.status === 'pending' && !sameSourceActivation(activation, sourceRecovery)))
) {
throw new Error('Invalid SSH PTY source activation identity')
}
return {
...(typeof result.replay === 'string' ? { replay: result.replay } : {}),
...(isPtyIncarnationId(result.incarnationId) ? { incarnationId: result.incarnationId } : {})
...(isPtyIncarnationId(result.incarnationId) ? { incarnationId: result.incarnationId } : {}),
...(sourceRecovery ? { sourceRecovery } : {}),
...(activation ? { sourceActivation: activation } : {})
}
}
export async function requestSshPtyAttach(args: {
mux: SshChannelMultiplexer
relayPtyId: string
params: Record<string, unknown>
timeoutMs?: number
commitSourceActivation?: boolean
installSourceActivation?: (
relayPtyId: string,
activation: PtySourceReceivingActivation
) => SshPtyReceivingActivationLease
rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void
}): Promise<SshPtyAttachResult> {
let activationLease: SshPtyReceivingActivationLease | undefined
const installFromResult = (result: SshPtyAttachResult): void => {
if (!activationLease && result.sourceActivation && args.installSourceActivation) {
activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation)
}
}
try {
const rawResult = await args.mux.request('pty.attach', args.params, {
...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }),
beforeResolve: (value) => installFromResult(parseSshPtyAttachResult(value))
})
const result = parseSshPtyAttachResult(rawResult)
installFromResult(result)
args.rememberPtyIncarnation?.(args.relayPtyId, result.incarnationId)
if (args.commitSourceActivation) {
activationLease?.commit()
}
return {
...result,
...(activationLease ? { sourceActivationLease: activationLease } : {})
}
} catch (error) {
activationLease?.rollback()
throw error
}
}
function parseSourceRecoveryResult(value: unknown): PtySourceRecoveryResult | undefined {
if (value === undefined) {
return undefined
}
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
throw new Error('Invalid SSH PTY source recovery response')
}
const input = value as Record<string, unknown>
if (input.status === 'restoreRequired' && typeof input.reason === 'string') {
return Object.freeze({ status: 'restoreRequired', reason: input.reason })
}
if (
input.status !== 'pending' ||
typeof input.deliveryToken !== 'string' ||
input.deliveryToken.length === 0 ||
typeof input.ptyIncarnation !== 'string' ||
input.ptyIncarnation.length === 0 ||
!positiveInteger(input.clientGeneration) ||
!positiveInteger(input.ownerGeneration) ||
!nonNegativeInteger(input.checkpointSourceEndSu) ||
!nonNegativeInteger(input.recoveryEndSu) ||
Number(input.recoveryEndSu) < Number(input.checkpointSourceEndSu)
) {
throw new Error('Invalid SSH PTY source recovery response')
}
return Object.freeze({
status: 'pending',
deliveryToken: input.deliveryToken,
ptyIncarnation: input.ptyIncarnation,
clientGeneration: Number(input.clientGeneration),
ownerGeneration: Number(input.ownerGeneration),
checkpointSourceEndSu: Number(input.checkpointSourceEndSu),
recoveryEndSu: Number(input.recoveryEndSu)
})
}
function positiveInteger(value: unknown): boolean {
return Number.isSafeInteger(value) && Number(value) > 0
}
function nonNegativeInteger(value: unknown): boolean {
return Number.isSafeInteger(value) && Number(value) >= 0
}
function sameSourceActivation(
left: PtySourceReceivingActivation,
right: PtySourceReceivingActivation
): boolean {
return (
left.clientGeneration === right.clientGeneration &&
left.ownerGeneration === right.ownerGeneration &&
left.ptyIncarnation === right.ptyIncarnation &&
left.deliveryToken === right.deliveryToken &&
left.checkpointSourceEndSu === right.checkpointSourceEndSu &&
left.recoveryEndSu === right.recoveryEndSu
)
}
export type { PtySourceRecoveryRequest }
export async function reattachSshPtySession(args: {
mux: SshChannelMultiplexer
connectionId: string
sessionId: string
options: PtySpawnOptions
}): Promise<PtySpawnResult> {
rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void
installSourceActivation?: (
relayPtyId: string,
activation: PtySourceReceivingActivation
) => SshPtyReceivingActivationLease
}): Promise<SshPtyReattachResult> {
const relaySessionId = toRelaySshPtyId(args.connectionId, args.sessionId)
console.warn(`[ssh-pty] spawn() called with sessionId=${args.sessionId}, attempting pty.attach`)
try {
// Why: expected pane identity prevents a reused relay id from attaching the wrong shell.
const expectedPaneKey = args.options.paneKey ?? args.options.env?.ORCA_PANE_KEY
const expectedTabId = args.options.tabId ?? args.options.env?.ORCA_TAB_ID
const attachResult = parseSshPtyAttachResult(
await args.mux.request('pty.attach', {
const attachResult = await requestSshPtyAttach({
mux: args.mux,
relayPtyId: relaySessionId,
params: {
id: relaySessionId,
cols: args.options.cols,
rows: args.options.rows,
suppressReplayNotification: true,
...(expectedPaneKey ? { expectedPaneKey } : {}),
...(expectedTabId ? { expectedTabId } : {})
})
)
},
installSourceActivation: args.installSourceActivation,
rememberPtyIncarnation: args.rememberPtyIncarnation
})
console.warn(
`[ssh-pty] pty.attach succeeded for ${args.sessionId}, replay=${!!attachResult.replay}`
)
@@ -65,7 +209,12 @@ export async function reattachSshPtySession(args: {
id: toAppSshPtyId(args.connectionId, relaySessionId),
isReattach: true,
...(attachResult.replay ? { replay: attachResult.replay } : {}),
...(attachResult.incarnationId ? { incarnationId: attachResult.incarnationId } : {})
...(attachResult.incarnationId ? { incarnationId: attachResult.incarnationId } : {}),
...(attachResult.sourceRecovery ? { sourceRecovery: attachResult.sourceRecovery } : {}),
...(attachResult.sourceActivation ? { sourceActivation: attachResult.sourceActivation } : {}),
...(attachResult.sourceActivationLease
? { sourceActivationLease: attachResult.sourceActivationLease }
: {})
}
} catch (error) {
// Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding.
@@ -84,10 +233,11 @@ export async function reattachSshPtySessionWithExitFence(
args: Parameters<typeof reattachSshPtySession>[0] & {
exitRaceTracker: SshPtySpawnExitRaceTracker
}
): Promise<PtySpawnResult> {
): Promise<SshPtyReattachResult> {
const operation = args.exitRaceTracker.begin()
let result: SshPtyReattachResult | undefined
try {
const result = await reattachSshPtySession(args)
result = await reattachSshPtySession(args)
const relayPtyId = toRelaySshPtyId(args.connectionId, result.id)
if (
args.exitRaceTracker.didMatchingExitArrive(operation, {
@@ -98,6 +248,9 @@ export async function reattachSshPtySessionWithExitFence(
throw new Error('agent_session_exited_during_start')
}
return result
} catch (error) {
result?.sourceActivationLease?.rollback()
throw error
} finally {
args.exitRaceTracker.finish(operation)
}
@@ -0,0 +1,44 @@
import { describe, expect, it, vi } from 'vitest'
import { SshPtySourceDeliveryLedger } from './ssh-pty-source-delivery-ledger'
describe('SshPtySourceDeliveryLedger', () => {
it('retains cancellation ownership when recovery transfer is superseded', async () => {
const request = vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
const ledger = new SshPtySourceDeliveryLedger({ request } as never, vi.fn())
const older = ledger.install(
'pty-1',
Object.freeze({
status: 'pending',
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-old',
checkpointSourceEndSu: 0,
recoveryEndSu: 0
})
)
ledger.install(
'pty-1',
Object.freeze({
status: 'pending',
clientGeneration: 3,
ownerGeneration: 4,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-new',
checkpointSourceEndSu: 0,
recoveryEndSu: 0
})
)
expect(() => older.transferToRecovery(vi.fn())).toThrow('ssh_source_receiving_activation_stale')
await expect(older.rollback()).resolves.toBe(true)
expect(request).toHaveBeenCalledOnce()
expect(request).toHaveBeenCalledWith('pty.cancelDelivery', {
id: 'pty-1',
clientGeneration: 2,
ownerGeneration: 3,
deliveryToken: 'token-old'
})
})
})
@@ -0,0 +1,320 @@
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
import type { SshPtySourceFrame } from './ssh-pty-source-frame'
export type PendingSshPtySourceData = Readonly<{
relayPtyId: string
params: Record<string, unknown>
data: string
source?: SshPtySourceFrame
}>
type SourceDeliveryLeaseState = {
phase: 'provisional' | 'recovery' | 'committing' | 'committed' | 'retired'
pendingData: PendingSshPtySourceData[]
recoverySink?: (pending: PendingSshPtySourceData) => void
exited: boolean
}
type SourceDeliveryState = Readonly<{
activation: PtySourceReceivingActivation
sourceEndSu: number
lease: SourceDeliveryLeaseState
previous?: SourceDeliveryState
}>
export type SshPtyRecoveryActivationLease = Readonly<{
commit: () => void
retire: () => void
}>
export type SshPtySourceDeliveryLease = Readonly<{
commit: () => void
rollback: () => Promise<boolean>
transferToRecovery: (
sink: (pending: PendingSshPtySourceData) => void
) => SshPtyRecoveryActivationLease
}>
export class SshPtySourceDeliveryLedger {
private readonly deliveryByPty = new Map<string, SourceDeliveryState>()
constructor(
private readonly mux: SshChannelMultiplexer,
private readonly publishData: (pending: PendingSshPtySourceData) => void
) {}
install(relayPtyId: string, activation: PtySourceReceivingActivation): SshPtySourceDeliveryLease {
if (!relayPtyId || activation.ptyIncarnation.length === 0) {
throw new Error('ssh_source_receiving_activation_invalid')
}
const previous = this.deliveryByPty.get(relayPtyId)
if (previous && sameReceivingActivation(previous.activation, activation)) {
if (previous.lease.phase !== 'committed') {
throw new Error('ssh_source_receiving_activation_stale')
}
return settledReceivingActivationLease()
}
if (
previous &&
(activation.clientGeneration <= previous.activation.clientGeneration ||
activation.ownerGeneration <= previous.activation.ownerGeneration ||
activation.deliveryToken === previous.activation.deliveryToken)
) {
throw new Error('ssh_source_receiving_activation_stale')
}
return this.installProvisional(relayPtyId, activation, previous)
}
admit(pending: PendingSshPtySourceData & { source: SshPtySourceFrame }): boolean {
const current = this.deliveryByPty.get(pending.relayPtyId)
if (!acceptsSourceFrame(current, pending.params, pending.source)) {
return false
}
const accepted = Object.freeze({
...current,
sourceEndSu: pending.source.sourceEndSu
}) as SourceDeliveryState
this.deliveryByPty.set(pending.relayPtyId, accepted)
if (accepted.lease.phase === 'recovery') {
accepted.lease.recoverySink?.(pending)
} else if (accepted.lease.phase !== 'committed') {
accepted.lease.pendingData.push(pending)
} else {
this.publishData(pending)
}
return true
}
recordExit(relayPtyId: string): void {
const current = this.deliveryByPty.get(relayPtyId)
if (current?.lease.phase === 'committed') {
this.deliveryByPty.delete(relayPtyId)
} else if (current) {
current.lease.exited = true
}
}
private installProvisional(
relayPtyId: string,
activation: PtySourceReceivingActivation,
previous: SourceDeliveryState | undefined
): SshPtySourceDeliveryLease {
const leaseState: SourceDeliveryLeaseState = {
phase: 'provisional',
pendingData: [],
exited: false
}
this.deliveryByPty.set(
relayPtyId,
Object.freeze({
activation,
sourceEndSu: activation.checkpointSourceEndSu,
lease: leaseState,
...(previous ? { previous } : {})
})
)
let settled = false
let transferInProgress = false
let rollbackSettlement: Promise<boolean> | undefined
return Object.freeze({
commit: () => {
if (settled || transferInProgress) {
return
}
settled = true
this.commit(relayPtyId, leaseState)
},
rollback: () => {
if (rollbackSettlement) {
return rollbackSettlement
}
if (settled || transferInProgress) {
return Promise.resolve(false)
}
settled = true
this.retire(relayPtyId, previous, leaseState)
rollbackSettlement = settleExactSourceDeliveryCancellation(this.mux, relayPtyId, activation)
return rollbackSettlement
},
transferToRecovery: (sink) => {
if (settled || transferInProgress || leaseState.phase !== 'provisional') {
throw new Error('ssh_source_receiving_activation_stale')
}
transferInProgress = true
try {
const recoveryLease = this.transferToRecovery(relayPtyId, leaseState, previous, sink)
settled = true
return recoveryLease
} finally {
transferInProgress = false
}
}
})
}
private transferToRecovery(
relayPtyId: string,
lease: SourceDeliveryLeaseState,
previous: SourceDeliveryState | undefined,
sink: (pending: PendingSshPtySourceData) => void
): SshPtyRecoveryActivationLease {
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
this.retire(relayPtyId, previous, lease)
throw new Error('ssh_source_receiving_activation_stale')
}
lease.phase = 'recovery'
lease.recoverySink = sink
try {
while (lease.pendingData.length > 0) {
sink(lease.pendingData.shift()!)
}
} catch (error) {
this.retire(relayPtyId, previous, lease)
throw error
}
let settled = false
return Object.freeze({
commit: () => {
if (settled) {
return
}
settled = true
lease.recoverySink = undefined
this.commit(relayPtyId, lease)
},
retire: () => {
if (settled) {
return
}
settled = true
lease.recoverySink = undefined
this.retire(relayPtyId, previous, lease)
}
})
}
private commit(relayPtyId: string, lease: SourceDeliveryLeaseState): void {
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
lease.phase = 'retired'
lease.pendingData.splice(0)
return
}
lease.phase = 'committing'
while (lease.pendingData.length > 0) {
this.publishData(lease.pendingData.shift()!)
}
lease.phase = 'committed'
const current = this.deliveryByPty.get(relayPtyId)
if (lease.exited && current?.lease === lease) {
this.deliveryByPty.delete(relayPtyId)
return
}
if (current?.lease === lease && current.previous) {
this.deliveryByPty.set(
relayPtyId,
Object.freeze({
activation: current.activation,
sourceEndSu: current.sourceEndSu,
lease: current.lease
})
)
}
}
private retire(
relayPtyId: string,
previous: SourceDeliveryState | undefined,
lease: SourceDeliveryLeaseState
): void {
lease.phase = 'retired'
lease.recoverySink = undefined
lease.pendingData.splice(0)
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
return
}
if (lease.exited) {
this.deliveryByPty.delete(relayPtyId)
return
}
const predecessor = activePredecessor(previous)
if (predecessor) {
this.deliveryByPty.set(relayPtyId, predecessor)
} else {
this.deliveryByPty.delete(relayPtyId)
}
}
}
function settledReceivingActivationLease(): SshPtySourceDeliveryLease {
return Object.freeze({
commit: () => {},
rollback: async () => true,
transferToRecovery: () => Object.freeze({ commit: () => {}, retire: () => {} })
})
}
function activePredecessor(previous?: SourceDeliveryState): SourceDeliveryState | undefined {
while (previous?.lease.phase === 'retired') {
previous = previous.previous
}
return previous
}
function sameReceivingActivation(
left: PtySourceReceivingActivation,
right: PtySourceReceivingActivation
): boolean {
return (
left.clientGeneration === right.clientGeneration &&
left.ownerGeneration === right.ownerGeneration &&
left.ptyIncarnation === right.ptyIncarnation &&
left.deliveryToken === right.deliveryToken &&
left.checkpointSourceEndSu === right.checkpointSourceEndSu &&
left.recoveryEndSu === right.recoveryEndSu
)
}
function acceptsSourceFrame(
current: SourceDeliveryState | undefined,
params: Record<string, unknown>,
source: SshPtySourceFrame
): current is SourceDeliveryState {
return Boolean(
current &&
current.lease.phase !== 'retired' &&
!current.lease.exited &&
current.activation.ptyIncarnation === params.ptyIncarnation &&
current.activation.deliveryToken === source.deliveryToken &&
current.activation.clientGeneration === source.clientGeneration &&
current.activation.ownerGeneration === source.ownerGeneration &&
current.sourceEndSu === source.sourceStartSu
)
}
async function settleExactSourceDeliveryCancellation(
mux: SshChannelMultiplexer,
relayPtyId: string,
activation: PtySourceReceivingActivation
): Promise<boolean> {
try {
const result = (await mux.request('pty.cancelDelivery', {
id: relayPtyId,
clientGeneration: activation.clientGeneration,
ownerGeneration: activation.ownerGeneration,
deliveryToken: activation.deliveryToken
})) as Record<string, unknown>
return (
result.canceled === true &&
nonNegativeSafeInteger(result.sentEndSu) &&
nonNegativeSafeInteger(result.creditedEndSu) &&
result.creditedEndSu <= result.sentEndSu
)
} catch {
return false
}
}
function nonNegativeSafeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) >= 0
}
@@ -0,0 +1,64 @@
import type { SshPtyDataCallback } from './ssh-pty-provider-contract'
export type SshPtySourceFrame = NonNullable<Parameters<SshPtyDataCallback>[0]['source']>
const SOURCE_KEYS = [
'deliveryToken',
'clientGeneration',
'ownerGeneration',
'sourceEndSu',
'sourceLengthSu',
'ptyIncarnation'
] as const
export function parseSshPtySourceFrame(
params: Record<string, unknown>,
data: string,
relayPtyId: string
): Readonly<{ source?: SshPtySourceFrame; malformed: boolean }> {
if (!SOURCE_KEYS.some((key) => params[key] !== undefined)) {
return Object.freeze({ malformed: false })
}
const sourceEndSu = params.sourceEndSu
const sourceLengthSu = params.sourceLengthSu
const rawLength = params.rawLength
const transformed = params.transformed === true
if (
typeof params.data !== 'string' ||
typeof params.deliveryToken !== 'string' ||
params.deliveryToken.length === 0 ||
!positiveSafeInteger(params.clientGeneration) ||
!positiveSafeInteger(params.ownerGeneration) ||
!nonNegativeSafeInteger(sourceEndSu) ||
!nonNegativeSafeInteger(sourceLengthSu) ||
sourceEndSu < sourceLengthSu ||
typeof params.ptyIncarnation !== 'string' ||
params.ptyIncarnation.length === 0 ||
(params.seq !== undefined && !nonNegativeSafeInteger(params.seq)) ||
(transformed ? rawLength !== sourceLengthSu : data.length !== sourceLengthSu) ||
(rawLength !== undefined && rawLength !== sourceLengthSu)
) {
return Object.freeze({ malformed: true })
}
const sourceStartSu = sourceEndSu - sourceLengthSu
return Object.freeze({
malformed: false,
source: Object.freeze({
relayPtyId,
spanId: `${params.deliveryToken}:${sourceStartSu}:${sourceEndSu}`,
clientGeneration: params.clientGeneration,
ownerGeneration: params.ownerGeneration,
deliveryToken: params.deliveryToken,
sourceStartSu,
sourceEndSu
})
})
}
function positiveSafeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) > 0
}
function nonNegativeSafeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) >= 0
}
+3
View File
@@ -22,6 +22,9 @@ export function buildSshPtySpawnEnv(args: {
merged.ORCA_RELAY_DIR = args.remoteCliBridgeEnv.relayDir
merged.ORCA_RELAY_NODE_PATH = args.remoteCliBridgeEnv.nodePath
merged.ORCA_RELAY_SOCKET_PATH = args.remoteCliBridgeEnv.sockPath
if (args.remoteCliBridgeEnv.credentialFile) {
merged.ORCA_RELAY_CREDENTIAL_FILE = args.remoteCliBridgeEnv.credentialFile
}
}
// Why: match local/daemon precedence—managed defaults cannot restore explicitly removed values.
for (const key of args.envToDelete ?? []) {
+137 -25
View File
@@ -16,6 +16,13 @@ import { parseFileUriPathParts } from '../daemon/osc7-file-uri'
import type { AgentStatus } from '../../shared/agent-detection'
import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges'
import type { TerminalOscColorQueryReplyColors } from '../../shared/terminal-osc-color-reply'
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
import type {
RemoteTerminalSourceRangeConsumerHooks,
RemoteTerminalSourceRangeReplacementPublication,
RemoteTerminalSourceRangeReplacementReservation,
RemoteTerminalSourceRangeStreamIdentity
} from './remote-terminal-source-range-consumer'
import {
createTerminalTitleTracker,
stripBrailleSpinnerGlyphs,
@@ -1431,6 +1438,19 @@ type RuntimeHeadlessTerminal = {
writeChain: Promise<void>
}
export type RuntimePtyDataAdmission = Readonly<{
sequence: number
completion: Promise<void>
}>
export type RuntimeTerminalDataMeta = Readonly<{
seq?: number
rawLength?: number
transformed?: boolean
cwd?: string
sourceRanges?: readonly TerminalOutputSourceRange[]
}>
type RuntimeVisibleTerminalState = {
lines: string[]
isAlternateScreen: boolean
@@ -2669,13 +2689,10 @@ export class OrcaRuntimeService {
// without polling. Keyed by ptyId for O(1) lookup per data event.
private dataListeners = new Map<
string,
Set<
(
data: string,
meta?: { seq?: number; rawLength?: number; transformed?: boolean; cwd?: string }
) => void
>
Set<(data: string, meta?: RuntimeTerminalDataMeta) => void>
>()
private remoteTerminalSourceRangeConsumerHooks: RemoteTerminalSourceRangeConsumerHooks | null =
null
// Why: startup draft paste can subscribe after the agent already emitted its
// ready marker. Keep a bounded raw buffer so fast startup output is replayed.
private recentPtyOutputById = new Map<string, RecentPtyOutputBuffer>()
@@ -8616,16 +8633,49 @@ export class OrcaRuntimeService {
}
}
resetPtyModelAfterMigrationFailure(ptyId: string): void {
this.providerSnapshotPreferredPtys.add(ptyId)
this.disposeHeadlessTerminal(ptyId)
}
/**
* Handles incoming data from a PTY process, running agent detection,
* updating terminal tail buffers, and triggering foreground agent refreshes.
*/
acceptPtyDataBounded(
ptyId: string,
data: string,
at: number,
sequenceChars = data.length,
transformed = false,
sourceRanges?: readonly TerminalOutputSourceRange[]
): RuntimePtyDataAdmission {
let completion: Promise<void> | null = null
const sequence = this.onPtyData(
ptyId,
data,
at,
sequenceChars,
transformed,
(receipt) => {
completion = receipt
},
sourceRanges
)
if (!completion) {
throw new Error('PTY model admission receipt was not captured')
}
return Object.freeze({ sequence, completion })
}
onPtyData(
ptyId: string,
data: string,
at: number,
sequenceChars = data.length,
transformed = false
transformed = false,
captureModelReceipt?: (completion: Promise<void>) => void,
sourceRanges?: readonly TerminalOutputSourceRange[]
): number {
const outputSequence = (this.ptyOutputSequenceById.get(ptyId) ?? 0) + sequenceChars
this.ptyOutputSequenceById.set(ptyId, outputSequence)
@@ -8664,7 +8714,13 @@ export class OrcaRuntimeService {
// applyTrackedPtyTitle) in byte order, superseding main's inline
// extractLastOscTitleForPty block (#7880/#7852 title/status semantics are
// preserved via the tracker + detectAgentStatusFromTitle path).
this.trackHeadlessTerminalData(ptyId, data, outputSequence, forwardQueryReplies)
const modelCompletion = this.trackHeadlessTerminalData(
ptyId,
data,
outputSequence,
forwardQueryReplies
)
captureModelReceipt?.(modelCompletion)
const pty = this.getOrCreatePtyWorktreeRecord(ptyId)
const ptyTailBefore = pty
@@ -8846,7 +8902,8 @@ export class OrcaRuntimeService {
seq: outputSequence,
rawLength: sequenceChars,
...(transformed ? { transformed: true } : {}),
...(cwdChanged && cwd !== null ? { cwd } : {})
...(cwdChanged && cwd !== null ? { cwd } : {}),
...(sourceRanges && sourceRanges.length > 0 ? { sourceRanges } : {})
}
for (const listener of listeners) {
try {
@@ -9721,14 +9778,71 @@ export class OrcaRuntimeService {
subscribeToTerminalData(
ptyId: string,
listener: (
data: string,
meta?: { seq?: number; rawLength?: number; transformed?: boolean; cwd?: string }
) => void
listener: (data: string, meta?: RuntimeTerminalDataMeta) => void
): () => void {
return addListenerToMap(this.dataListeners, ptyId, listener)
}
setRemoteTerminalSourceRangeConsumerHooks(
hooks: RemoteTerminalSourceRangeConsumerHooks | null
): void {
this.remoteTerminalSourceRangeConsumerHooks = hooks
}
attachRemoteTerminalSourceRangeConsumer(
identity: RemoteTerminalSourceRangeStreamIdentity
): boolean {
return this.remoteTerminalSourceRangeConsumerHooks?.attach(identity) ?? false
}
settleRemoteTerminalSourceRanges(
identity: RemoteTerminalSourceRangeStreamIdentity,
ranges: readonly TerminalOutputSourceRange[]
): void {
this.remoteTerminalSourceRangeConsumerHooks?.settle(identity, ranges)
}
reserveRemoteTerminalSourceRangeReplacement(
identity: RemoteTerminalSourceRangeStreamIdentity,
requiredSeq: number,
reason: string
): RemoteTerminalSourceRangeReplacementReservation | null {
return (
this.remoteTerminalSourceRangeConsumerHooks?.reserveReplacement(
identity,
requiredSeq,
reason
) ?? null
)
}
commitRemoteTerminalSourceRangeReplacement(
reservation: RemoteTerminalSourceRangeReplacementReservation,
publication: RemoteTerminalSourceRangeReplacementPublication
): boolean {
return (
this.remoteTerminalSourceRangeConsumerHooks?.commitReplacement(reservation, publication) ??
false
)
}
rollbackRemoteTerminalSourceRangeReplacement(
reservation: RemoteTerminalSourceRangeReplacementReservation,
reason: string
): boolean {
return (
this.remoteTerminalSourceRangeConsumerHooks?.rollbackReplacement(reservation, reason) ?? false
)
}
cancelRemoteTerminalSourceRanges(
identity: RemoteTerminalSourceRangeStreamIdentity,
ranges: readonly TerminalOutputSourceRange[],
reason: string
): void {
this.remoteTerminalSourceRangeConsumerHooks?.cancel(identity, ranges, reason)
}
/** Set by pty IPC: fires when a PTY gains/loses remote view subscribers so
* the daemon background mark (keep-tail stream thinning) can resync a
* live mobile/web view consumes raw bytes and must never be thinned, even
@@ -10186,19 +10300,17 @@ export class OrcaRuntimeService {
data: string,
outputSequence: number,
forwardQueryReplies = false
): void {
): Promise<void> {
const state = this.getOrCreateHeadlessTerminal(ptyId)
state.writeChain = state.writeChain
.then(async () => {
// Why: the ingestion-time ownership decision is closed over this
// chain link; async scheduling cannot retroactively change it.
await state.emulator.write(data, { forwardQueryReplies })
state.outputSequence = outputSequence
})
.catch(() => {
// Best-effort state tracking; live streaming must continue even if
// xterm rejects a malformed or raced write during shutdown.
})
const completion = state.writeChain.then(async () => {
// Why: the ingestion-time ownership decision is closed over this
// chain link; async scheduling cannot retroactively change it.
await state.emulator.write(data, { forwardQueryReplies })
state.outputSequence = outputSequence
})
// Legacy callers remain best-effort; bounded SSH admission observes the raw receipt.
state.writeChain = completion.catch(() => {})
return completion
}
/** Shared factory for the per-PTY runtime emulators (seed, hydration, and
@@ -0,0 +1,44 @@
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
export type RemoteTerminalSourceRangeStreamIdentity = Readonly<{
ptyId: string
consumerId: string
streamGeneration: string
}>
export type RemoteTerminalSourceRangeReplacementReservation = Readonly<{
reservationId: string
identity: RemoteTerminalSourceRangeStreamIdentity
requiredSeq: number
}>
export type RemoteTerminalSourceRangeReplacementPublication = Readonly<{
source: 'headless' | 'renderer'
seq: number
}>
export type RemoteTerminalSourceRangeConsumerHooks = {
attach: (identity: RemoteTerminalSourceRangeStreamIdentity) => boolean
settle: (
identity: RemoteTerminalSourceRangeStreamIdentity,
ranges: readonly TerminalOutputSourceRange[]
) => void
reserveReplacement: (
identity: RemoteTerminalSourceRangeStreamIdentity,
requiredSeq: number,
reason: string
) => RemoteTerminalSourceRangeReplacementReservation | null
commitReplacement: (
reservation: RemoteTerminalSourceRangeReplacementReservation,
publication: RemoteTerminalSourceRangeReplacementPublication
) => boolean
rollbackReplacement: (
reservation: RemoteTerminalSourceRangeReplacementReservation,
reason: string
) => boolean
cancel: (
identity: RemoteTerminalSourceRangeStreamIdentity,
ranges: readonly TerminalOutputSourceRange[],
reason: string
) => void
}
+378 -82
View File
@@ -1,5 +1,6 @@
/* oxlint-disable max-lines -- Why: terminal RPC methods are co-located for discoverability; splitting would scatter related handlers across files. */
import { z } from 'zod'
import { randomUUID } from 'node:crypto'
import {
InvalidArgumentError,
defineMethod,
@@ -19,6 +20,7 @@ import {
} from '../../../../shared/terminal-stream-protocol'
import {
iterateTerminalOutputFrameChunks,
sliceTerminalOutputSourceRanges,
type TerminalOutputFrameChunk,
type TerminalOutputMeta
} from '../terminal-output-frame-chunks'
@@ -61,6 +63,13 @@ import {
TERMINAL_OUTPUT_BATCH_MAX_BYTES
} from '../../../../shared/terminal-multiplex-flow-control'
import { drainTerminalMultiplexRoundRobin } from '../terminal-multiplex-round-robin'
import type { TerminalSourceRangeLedger } from '../terminal-source-range-ledger'
import { TerminalSourceRangeRegistry } from '../terminal-source-range-registry'
import {
sameTerminalOutputSourceIdentity,
type TerminalOutputSourceRange
} from '../../../../shared/terminal-output-source-range'
import type { RemoteTerminalSourceRangeReplacementReservation } from '../../remote-terminal-source-range-consumer'
const REQUESTED_SNAPSHOT_BYTE_BUDGET = 2 * 1024 * 1024
const TERMINAL_OUTPUT_FLUSH_MS = 5
@@ -112,6 +121,11 @@ type TerminalMultiplexStream = {
client: TerminalViewportClient | undefined
isMobile: boolean
ackOutput: boolean
ackOutputSourceRanges: boolean
streamGeneration: string
sourceRangeLedger: TerminalSourceRangeLedger | null
sourceRangeConsumerAttached: boolean
sourceRangeReplacement: RemoteTerminalSourceRangeReplacementReservation | null
ackInFlightBytes: number
ackWindowBytes: number
supportsDesktopViewportClaims: boolean
@@ -157,6 +171,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
let lastSeq: number | undefined
let pendingCwd: string | undefined
let pendingRawLength = 0
let pendingSourceRanges: TerminalOutputSourceRange[] = []
let timer: ReturnType<typeof setTimeout> | null = null
const clearTimer = (): void => {
@@ -174,10 +189,13 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
}
const data = chunks.length === 1 ? chunks[0]! : chunks.join('')
const meta =
typeof lastSeq === 'number' || pendingCwd !== undefined
typeof lastSeq === 'number' || pendingCwd !== undefined || pendingSourceRanges.length > 0
? {
...(typeof lastSeq === 'number' ? { seq: lastSeq, rawLength: pendingRawLength } : {}),
...(pendingCwd !== undefined ? { cwd: pendingCwd } : {})
...(pendingCwd !== undefined ? { cwd: pendingCwd } : {}),
...(pendingSourceRanges.length > 0
? { sourceRanges: Object.freeze(pendingSourceRanges.slice()) }
: {})
}
: undefined
chunks = []
@@ -185,6 +203,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
lastSeq = undefined
pendingCwd = undefined
pendingRawLength = 0
pendingSourceRanges = []
onFlush(data, meta)
}
@@ -199,12 +218,26 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
onFlush(data, { ...meta, rawLength, transformed: true })
return
}
const nextSourceRanges = meta?.sourceRanges ?? []
const lastSourceRange = pendingSourceRanges.at(-1)
const firstNextSourceRange = nextSourceRanges[0]
if (
chunks.length > 0 &&
(pendingSourceRanges.length > 0 !== nextSourceRanges.length > 0 ||
(lastSourceRange &&
firstNextSourceRange &&
(!sameTerminalOutputSourceIdentity(lastSourceRange, firstNextSourceRange) ||
lastSourceRange.displayEnd !== firstNextSourceRange.displayStart)))
) {
flush()
}
if (meta?.cwd !== undefined) {
flush()
pendingCwd = meta.cwd
}
chunks.push(data)
pendingRawLength += rawLength
pendingSourceRanges.push(...nextSourceRanges)
const remainingBudget = Math.max(1, TERMINAL_OUTPUT_BATCH_MAX_BYTES - bytes)
const measurement = measureTerminalStreamByteLength(data, {
stopAfterBytes: remainingBudget
@@ -231,6 +264,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
chunks = []
bytes = 0
pendingRawLength = 0
pendingSourceRanges = []
}
}
}
@@ -380,22 +414,38 @@ function appendPendingMultiplexOutput(
function getOutputAfterSnapshotSeq(
chunk: TerminalOutputChunk,
snapshotSeq: number | undefined
): string | null {
): TerminalOutputChunk | null {
if (
typeof snapshotSeq !== 'number' ||
typeof chunk.meta?.seq !== 'number' ||
typeof chunk.meta.rawLength !== 'number'
) {
return chunk.data
return chunk
}
if (chunk.meta.seq <= snapshotSeq) {
return null
}
const chunkStartSeq = chunk.meta.seq - chunk.meta.rawLength
if (chunkStartSeq >= snapshotSeq) {
return chunk.data
return chunk
}
if (chunk.meta.transformed) {
return null
}
const offset = snapshotSeq - chunkStartSeq
return {
data: chunk.data.slice(offset),
bytes: chunk.bytes,
meta: {
...chunk.meta,
rawLength: chunk.meta.rawLength - offset,
sourceRanges: sliceTerminalOutputSourceRanges(
chunk.meta.sourceRanges,
offset,
chunk.data.length
)
}
}
return chunk.data.slice(snapshotSeq - chunkStartSeq)
}
function stripSnapshotBoundaryQuerySuffixes(
@@ -551,36 +601,45 @@ async function serializeBudgetedRequestedSnapshot(
}
function sendSnapshotFrames(
sendFrame: (opcode: TerminalStreamOpcode, payload?: Uint8Array<ArrayBufferLike>) => void,
sendFrame: (
opcode: TerminalStreamOpcode,
payload?: Uint8Array<ArrayBufferLike>
) => boolean | void,
options: SnapshotFrameOptions
): { bytes: number; chunks: number } {
sendFrame(
TerminalStreamOpcode.SnapshotStart,
encodeTerminalStreamJson({
kind: options.kind,
cols: options.cols,
rows: options.rows,
requestId: options.requestId,
displayMode: options.displayMode,
reason: options.reason,
seq: options.seq,
cwd: options.cwd,
source: options.source,
oscLinks: options.oscLinks,
pendingEscapeTailAnsi: options.pendingEscapeTailAnsi,
truncated: options.truncated === true,
truncatedByByteBudget: options.truncatedByByteBudget === true
})
)
): { bytes: number; chunks: number; published: boolean } {
if (
sendFrame(
TerminalStreamOpcode.SnapshotStart,
encodeTerminalStreamJson({
kind: options.kind,
cols: options.cols,
rows: options.rows,
requestId: options.requestId,
displayMode: options.displayMode,
reason: options.reason,
seq: options.seq,
cwd: options.cwd,
source: options.source,
oscLinks: options.oscLinks,
pendingEscapeTailAnsi: options.pendingEscapeTailAnsi,
truncated: options.truncated === true,
truncatedByByteBudget: options.truncatedByByteBudget === true
})
) === false
) {
return { bytes: 0, chunks: 0, published: false }
}
let chunks = 0
let bytes = 0
for (const chunk of iterateTerminalStreamTextPayloads(options.data)) {
if (sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk) === false) {
return { bytes, chunks, published: false }
}
chunks++
bytes += chunk.byteLength
sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk)
}
sendFrame(TerminalStreamOpcode.SnapshotEnd)
return { bytes, chunks }
const published = sendFrame(TerminalStreamOpcode.SnapshotEnd) !== false
return { bytes, chunks, published }
}
async function serializeBudgetedMobileSnapshot(
@@ -953,14 +1012,24 @@ const TerminalMultiplexSubscribeFrame = TerminalHandle.extend({
capabilities: z
.object({
ackOutput: z.literal(1).optional(),
ackOutputSourceRanges: z.literal(1).optional(),
desktopViewportClaims: z.literal(1).optional()
})
.optional()
})
const TerminalMultiplexAckFrame = z.object({
bytes: z.number().int().nonnegative()
})
const TerminalMultiplexLegacyAckFrame = z
.object({
bytes: z.number().int().nonnegative()
})
.strict()
const TerminalMultiplexSourceRangeAckFrame = z
.object({
streamGeneration: z.string().min(1),
ackedEndByte: z.number().int().nonnegative()
})
.strict()
const TerminalMultiplexSnapshotRequestFrame = z.object({
requestId: z.number().int().positive().optional(),
@@ -1514,6 +1583,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
let closed = false
let cursor = 0
const streams = new Map<number, TerminalMultiplexStream>()
const sourceRangeRegistry = new TerminalSourceRangeRegistry()
const pendingPtyWaitControllers = new Map<number, Set<AbortController>>()
let ackTotalInFlightBytes = 0
let ackTotalWindowBytes = TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES
@@ -1526,9 +1596,11 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
streamId: number,
opcode: TerminalStreamOpcode,
payload: Uint8Array<ArrayBufferLike> = new Uint8Array(),
seq?: number
seq?: number,
onRejected?: () => void
): boolean => {
if (closed) {
onRejected?.()
return false
}
// Why: a seq-less Output chunk must carry sentinel 0, not the control-frame cursor, or it poisons the client's frame-drop tracker.
@@ -1540,10 +1612,12 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
encodeTerminalStreamFrame({ opcode, streamId, seq: resolvedSeq, payload })
)
} catch {
onRejected?.()
closeMultiplex()
return false
}
if (sent === false) {
onRejected?.()
// Why: false means the transport discarded this frame; reconnect is the only available retry boundary with an authoritative snapshot.
closeMultiplex()
return false
@@ -1577,22 +1651,43 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
}
return (
stream.ackInFlightBytes + bytes <= stream.ackWindowBytes &&
ackTotalInFlightBytes + bytes <= ackTotalWindowBytes
ackTotalInFlightBytes + bytes <= ackTotalWindowBytes &&
(!stream.ackOutputSourceRanges || stream.sourceRangeLedger?.canAccept(bytes) === true)
)
}
const sendAckGatedOutput = (
stream: TerminalMultiplexStream,
chunk: TerminalOutputFrameChunk
): boolean => {
const prepared = stream.ackOutputSourceRanges
? stream.sourceRangeLedger?.prepareAccept(
chunk.bytes.byteLength,
chunk.displayLength,
chunk.sourceRanges ?? [],
chunk.seq
)
: undefined
if (stream.ackOutputSourceRanges && prepared?.status !== 'ready') {
if (prepared?.status !== 'capacity') {
detachStream(stream.streamId, true)
}
return false
}
const admission = prepared?.status === 'ready' ? prepared.admission : undefined
const sent = sendFrame(
stream.streamId,
chunk.opcode ?? TerminalStreamOpcode.Output,
chunk.bytes,
chunk.seq
chunk.seq,
admission?.rollback
)
if (!sent) {
return false
}
if (admission && !admission.commit()) {
detachStream(stream.streamId, true)
return false
}
if (stream.ackOutput) {
stream.ackInFlightBytes += chunk.bytes.byteLength
ackTotalInFlightBytes += chunk.bytes.byteLength
@@ -1625,6 +1720,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
return
}
stream.ackRecoverySnapshotInFlight = true
let replacement: RemoteTerminalSourceRangeReplacementReservation | null = null
try {
const serialized = await serializeBudgetedRequestedSnapshot(runtime, stream.ptyId, 0)
if (closed || streams.get(stream.streamId) !== stream) {
@@ -1633,21 +1729,75 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
if (!serialized) {
throw new Error('Remote terminal recovery snapshot unavailable.')
}
if (
stream.ackOutputSourceRanges &&
(serialized.source === undefined || typeof serialized.seq !== 'number')
) {
throw new Error('Remote terminal recovery snapshot source identity unavailable.')
}
if (
stream.ackOutputSourceRanges &&
serialized.source !== undefined &&
typeof serialized.seq === 'number'
) {
replacement = runtime.reserveRemoteTerminalSourceRangeReplacement(
{
ptyId: stream.ptyId,
consumerId: stream.remoteDesktopSubscriptionKey,
streamGeneration: stream.streamGeneration
},
serialized.seq,
'ack-pending-overflow'
)
stream.sourceRangeReplacement = replacement
}
const displayMode = runtime.getMobileDisplayMode(stream.ptyId)
// Why: dropped ACK-pending output breaks live replay; send a fresh snapshot before resuming output.
sendSnapshotFrames((opcode, payload) => sendFrame(stream.streamId, opcode, payload), {
kind: 'scrollback',
cols: serialized.cols,
rows: serialized.rows,
displayMode,
reason: 'ack-pending-overflow',
seq: serialized.seq,
source: serialized.source,
truncatedByByteBudget: serialized.truncatedByByteBudget,
data: serialized.data
})
const publication = sendSnapshotFrames(
(opcode, payload) =>
!closed &&
streams.get(stream.streamId) === stream &&
sendFrame(stream.streamId, opcode, payload),
{
kind: 'scrollback',
cols: serialized.cols,
rows: serialized.rows,
displayMode,
reason: 'ack-pending-overflow',
seq: serialized.seq,
source: serialized.source,
truncatedByByteBudget: serialized.truncatedByByteBudget,
data: serialized.data
}
)
if (!publication.published) {
throw new Error('Remote terminal recovery snapshot was not published.')
}
if (closed || streams.get(stream.streamId) !== stream) {
throw new Error('Remote terminal recovery snapshot stream detached.')
}
const localReplacement = replacement
? typeof serialized.seq === 'number'
? stream.sourceRangeLedger?.planSourceRangeReplacement(serialized.seq)
: null
: null
if (replacement && !localReplacement) {
throw new Error('Remote terminal recovery source ledger replacement unavailable.')
}
if (
replacement &&
(!serialized.source ||
typeof serialized.seq !== 'number' ||
!runtime.commitRemoteTerminalSourceRangeReplacement(replacement, {
source: serialized.source,
seq: serialized.seq
}))
) {
throw new Error('Remote terminal recovery snapshot replacement was not accepted.')
}
localReplacement?.commit()
stream.sourceRangeReplacement = null
replacement = null
if (typeof serialized.seq === 'number') {
// Why: chunks queued before the snapshot serialized are already in it; replaying them would duplicate output.
const snapshotSeq = serialized.seq
const retained = stream.ackPendingOutput.filter(
(chunk) => !(typeof chunk.seq === 'number' && chunk.seq <= snapshotSeq)
@@ -1660,11 +1810,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
}
stream.ackPendingOutputOverflowed = false
} catch (error) {
if (replacement) {
if (stream.sourceRangeReplacement === replacement) {
stream.sourceRangeReplacement = null
runtime.rollbackRemoteTerminalSourceRangeReplacement(
replacement,
'ack-pending-overflow-unpublished'
)
}
replacement = null
}
if (closed || streams.get(stream.streamId) !== stream) {
return
}
sendStreamError(
stream.streamId,
error instanceof Error ? error.message : 'Remote terminal recovery snapshot failed.'
)
// Why: retrying the same failed recovery from finally creates an unbounded error loop.
detachStream(stream.streamId, true)
} finally {
if (streams.get(stream.streamId) === stream) {
@@ -1735,6 +1897,56 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - acknowledged)
flushAllAckPendingOutput()
}
const acknowledgeSourceRanges = (
stream: TerminalMultiplexStream,
streamGeneration: string,
ackedEndByte: number
): void => {
if (!stream.ackOutputSourceRanges) {
return
}
const result = stream.sourceRangeLedger?.acknowledge(streamGeneration, ackedEndByte)
if (!result) {
return
}
if (result.status !== 'accepted') {
return
}
if (result.settled.length > 0) {
runtime.settleRemoteTerminalSourceRanges(
{
ptyId: stream.ptyId,
consumerId: stream.remoteDesktopSubscriptionKey,
streamGeneration: stream.streamGeneration
},
result.settled
)
}
acknowledgeOutput(stream, result.acknowledgedBytes)
}
const detachSourceRangeConsumer = (stream: TerminalMultiplexStream, reason: string): void => {
if (!stream.sourceRangeConsumerAttached) {
return
}
stream.sourceRangeConsumerAttached = false
const ledger = stream.sourceRangeLedger
stream.sourceRangeLedger = null
if (!ledger) {
return
}
const identity = {
ptyId: stream.ptyId,
consumerId: stream.remoteDesktopSubscriptionKey,
streamGeneration: stream.streamGeneration
}
const transfer = ledger.beginTransfer()
const ranges = transfer.frames.flatMap((frame) => frame.sourceRanges)
try {
runtime.cancelRemoteTerminalSourceRanges(identity, ranges, reason)
} finally {
transfer.commit()
}
}
const detachStream = (
streamId: number,
emitEnd: boolean,
@@ -1744,8 +1956,17 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
if (!stream) {
return
}
const replacement = stream.sourceRangeReplacement
stream.sourceRangeReplacement = null
if (replacement) {
runtime.rollbackRemoteTerminalSourceRangeReplacement(
replacement,
'stream-detached-replacement-aborted'
)
}
stream.outputBatcher.flush()
stream.outputBatcher.dispose()
detachSourceRangeConsumer(stream, 'stream-detached')
ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - stream.ackInFlightBytes)
stream.ackInFlightBytes = 0
stream.ackPendingOutput = []
@@ -1827,11 +2048,21 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
return
}
if (frame.opcode === TerminalStreamOpcode.Ack) {
const parsed = TerminalMultiplexAckFrame.safeParse(
decodeTerminalStreamJson<unknown>(frame.payload) ?? {}
)
if (parsed.success) {
acknowledgeOutput(stream, parsed.data.bytes)
const payload = decodeTerminalStreamJson<unknown>(frame.payload) ?? {}
if (stream.ackOutputSourceRanges) {
const parsed = TerminalMultiplexSourceRangeAckFrame.safeParse(payload)
if (parsed.success) {
acknowledgeSourceRanges(
stream,
parsed.data.streamGeneration,
parsed.data.ackedEndByte
)
}
} else {
const parsed = TerminalMultiplexLegacyAckFrame.safeParse(payload)
if (parsed.success) {
acknowledgeOutput(stream, parsed.data.bytes)
}
}
return
}
@@ -2024,12 +2255,12 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
// high-water, so covered bytes would render twice; tagged
// snapshots feed a side consumer and the live view still
// needs every buffered chunk.
const uncoveredData =
const uncovered =
typeof requestId === 'number'
? chunk.data
? chunk
: getOutputAfterSnapshotSeq(chunk, sentSnapshotOutputSeq)
if (uncoveredData) {
stream.outputBatcher.push(uncoveredData, chunk.meta)
if (uncovered) {
stream.outputBatcher.push(uncovered.data, uncovered.meta)
}
}
}
@@ -2140,6 +2371,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
detachStream(request.streamId, false)
const ptyId = leaf.ptyId
const remoteDesktopSubscriptionKey = `multiplex:${connectionId}:${request.streamId}`
const streamGeneration = randomUUID()
const requestedSourceRangeConsumer =
request.capabilities?.ackOutput === 1 && request.capabilities?.ackOutputSourceRanges === 1
const sourceRangeLedger = requestedSourceRangeConsumer
? sourceRangeRegistry.open(streamGeneration)
: null
const sourceRangeConsumerAttached =
sourceRangeLedger !== null &&
runtime.attachRemoteTerminalSourceRangeConsumer({
ptyId,
consumerId: remoteDesktopSubscriptionKey,
streamGeneration
})
if (!sourceRangeConsumerAttached) {
sourceRangeLedger?.close()
}
const stream: TerminalMultiplexStream = {
streamId: request.streamId,
terminal: request.terminal,
@@ -2147,13 +2395,18 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
client: request.client,
isMobile,
ackOutput: request.capabilities?.ackOutput === 1,
ackOutputSourceRanges: sourceRangeConsumerAttached,
streamGeneration,
sourceRangeLedger: sourceRangeConsumerAttached ? sourceRangeLedger : null,
sourceRangeConsumerAttached,
sourceRangeReplacement: null,
ackInFlightBytes: 0,
ackWindowBytes: TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES,
supportsDesktopViewportClaims: request.capabilities?.desktopViewportClaims === 1,
desktopClaimTail: Promise.resolve(true),
registeredRemoteDesktopDriver: false,
// Why: streamId is client-local, so key the width floor by connectionId or two connections sharing stream 1 for one PTY clobber each other's floor.
remoteDesktopSubscriptionKey: `multiplex:${connectionId}:${request.streamId}`,
remoteDesktopSubscriptionKey,
pendingRemoteDesktopViewport: null,
buffering: true,
ackPendingOutput: [],
@@ -2273,35 +2526,77 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
rows: serialized?.rows ?? size?.rows,
displayMode,
seq: layoutSeq,
...(stream.ackOutputSourceRanges
? {
streamGeneration: stream.streamGeneration,
capabilities: { ackOutputSourceRanges: 1 as const }
}
: {}),
truncated:
initialOutputOverflowed ||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read))
})
sendSnapshotFrames((opcode, payload) => sendFrame(request.streamId, opcode, payload), {
kind: 'scrollback',
cols: serialized?.cols ?? size?.cols ?? 80,
rows: serialized?.rows ?? size?.rows ?? 24,
displayMode,
seq: snapshotFrameSeq,
cwd: serialized?.cwd,
truncated:
initialOutputOverflowed ||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
truncatedByByteBudget: serialized?.truncatedByByteBudget,
source: serialized?.source,
oscLinks: serialized?.oscLinks,
pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi,
data: serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '')
})
stream.sourceRangeReplacement =
stream.ackOutputSourceRanges &&
serialized?.source !== undefined &&
typeof serialized.seq === 'number'
? runtime.reserveRemoteTerminalSourceRangeReplacement(
{
ptyId,
consumerId: stream.remoteDesktopSubscriptionKey,
streamGeneration: stream.streamGeneration
},
serialized.seq,
'initial-snapshot'
)
: null
const snapshotPublication = sendSnapshotFrames(
(opcode, payload) => sendFrame(request.streamId, opcode, payload),
{
kind: 'scrollback',
cols: serialized?.cols ?? size?.cols ?? 80,
rows: serialized?.rows ?? size?.rows ?? 24,
displayMode,
seq: snapshotFrameSeq,
cwd: serialized?.cwd,
truncated:
initialOutputOverflowed ||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
truncatedByByteBudget: serialized?.truncatedByByteBudget,
source: serialized?.source,
oscLinks: serialized?.oscLinks,
pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi,
data:
serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '')
}
)
const replacement = stream.sourceRangeReplacement
stream.sourceRangeReplacement = null
if (replacement) {
const committed =
snapshotPublication.published &&
serialized?.source !== undefined &&
typeof serialized.seq === 'number' &&
runtime.commitRemoteTerminalSourceRangeReplacement(replacement, {
source: serialized.source,
seq: serialized.seq
})
if (!committed) {
runtime.rollbackRemoteTerminalSourceRangeReplacement(
replacement,
'initial-snapshot-unpublished'
)
}
}
// Why: baseline for resize re-stream gating; the client already rewrapped to these cols via the initial snapshot replay.
stream.lastResizeCols = serialized?.cols ?? size?.cols
stream.buffering = false
const pendingOutput = stream.pendingOutput.splice(0)
if (!initialOutputOverflowed) {
for (const chunk of pendingOutput) {
const uncoveredData = getOutputAfterSnapshotSeq(chunk, snapshotOutputSeq)
if (uncoveredData) {
stream.outputBatcher.push(uncoveredData, chunk.meta)
const uncovered = getOutputAfterSnapshotSeq(chunk, snapshotOutputSeq)
if (uncovered) {
stream.outputBatcher.push(uncovered.data, uncovered.meta)
}
}
}
@@ -2982,9 +3277,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
if (stableRendererSnapshot?.data.length) {
serialized = stableRendererSnapshot
const trailingOutput = pendingOutput.flatMap((item) => {
const data = getOutputAfterSnapshotSeq(item, stableRendererSnapshot.seq)
const output = getOutputAfterSnapshotSeq(item, stableRendererSnapshot.seq)
const seq = item.meta?.seq
return data && typeof seq === 'number' ? [{ data, seq }] : []
return output && typeof seq === 'number' ? [{ data: output.data, seq }] : []
})
runtime.replaceHeadlessTerminalFromRendererSnapshotForRecovery(
ptyId,
@@ -3118,8 +3413,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
}
if (!initialOutputOverflowed) {
for (const item of bufferedOutput) {
let uncoveredData = getOutputAfterSnapshotSeq(item, snapshotOutputSeq)
let uncoveredMeta = item.meta
const uncovered = getOutputAfterSnapshotSeq(item, snapshotOutputSeq)
let uncoveredData = uncovered?.data ?? null
let uncoveredMeta = uncovered?.meta
if (
uncoveredData &&
uncoveredData !== item.data &&
File diff suppressed because it is too large Load Diff
@@ -47,12 +47,13 @@ function* legacyIterateTerminalOutputFrameChunks(
yield {
opcode: TerminalStreamOpcode.OutputSpan,
bytes: encodeTerminalStreamJson({ data, rawLength, transformed: true }),
displayLength: data.length,
seq: meta?.seq
}
return
}
if (!legacyByteLengthExceeds(data, TERMINAL_STREAM_CHUNK_BYTES)) {
yield { bytes: encodeTerminalStreamText(data), seq: meta?.seq }
yield { bytes: encodeTerminalStreamText(data), displayLength: data.length, seq: meta?.seq }
return
}
const canPreserveChunkSeq = typeof meta?.seq === 'number' && rawLength === data.length
@@ -83,11 +84,18 @@ function* legacyIterateTerminalOutputFrameChunks(
if (nextChunk) {
if (shouldDelayFinalSeq) {
if (delayedChunk) {
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
yield {
bytes: encodeTerminalStreamText(delayedChunk.text),
displayLength: delayedChunk.text.length
}
}
delayedChunk = nextChunk
} else {
yield { bytes: encodeTerminalStreamText(nextChunk.text), seq: nextChunk.seq }
yield {
bytes: encodeTerminalStreamText(nextChunk.text),
displayLength: nextChunk.text.length,
seq: nextChunk.seq
}
}
}
}
@@ -99,17 +107,28 @@ function* legacyIterateTerminalOutputFrameChunks(
if (shouldDelayFinalSeq) {
if (finalChunk) {
if (delayedChunk) {
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
yield {
bytes: encodeTerminalStreamText(delayedChunk.text),
displayLength: delayedChunk.text.length
}
}
delayedChunk = finalChunk
}
if (delayedChunk) {
yield { bytes: encodeTerminalStreamText(delayedChunk.text), seq: meta.seq }
yield {
bytes: encodeTerminalStreamText(delayedChunk.text),
displayLength: delayedChunk.text.length,
seq: meta.seq
}
}
return
}
if (finalChunk) {
yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: finalChunk.seq }
yield {
bytes: encodeTerminalStreamText(finalChunk.text),
displayLength: finalChunk.text.length,
seq: finalChunk.seq
}
}
}
@@ -4,6 +4,7 @@ import {
encodeTerminalStreamText
} from '../../../shared/terminal-stream-protocol'
import { TERMINAL_STREAM_CHUNK_BYTES } from '../../../shared/terminal-multiplex-flow-control'
import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range'
import { terminalStreamByteLength } from './terminal-stream-byte-length'
export type TerminalOutputMeta = {
@@ -11,12 +12,15 @@ export type TerminalOutputMeta = {
rawLength?: number
transformed?: boolean
cwd?: string
sourceRanges?: readonly TerminalOutputSourceRange[]
}
export type TerminalOutputFrameChunk = {
bytes: Uint8Array<ArrayBufferLike>
displayLength: number
seq?: number
opcode?: TerminalStreamOpcode
sourceRanges?: readonly TerminalOutputSourceRange[]
}
export const TERMINAL_STREAM_BYTE_PROBE_CODE_UNITS = 8 * 1024
@@ -55,12 +59,19 @@ export function* iterateTerminalOutputFrameChunks(
yield {
opcode: TerminalStreamOpcode.OutputSpan,
bytes: encodeTerminalStreamJson({ data, rawLength, transformed: true }),
seq: meta?.seq
displayLength: data.length,
seq: meta?.seq,
sourceRanges: meta?.sourceRanges
}
return
}
if (!exceedsTerminalStreamChunkBytes(data)) {
yield { bytes: encodeTerminalStreamText(data), seq: meta?.seq }
yield {
bytes: encodeTerminalStreamText(data),
displayLength: data.length,
seq: meta?.seq,
sourceRanges: meta?.sourceRanges
}
return
}
const canPreserveChunkSeq = typeof meta?.seq === 'number' && rawLength === data.length
@@ -109,11 +120,23 @@ export function* iterateTerminalOutputFrameChunks(
const nextChunk = takeChunk(index)
if (shouldDelayFinalSeq) {
if (delayedChunk) {
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
yield {
bytes: encodeTerminalStreamText(delayedChunk.text),
displayLength: delayedChunk.text.length
}
}
delayedChunk = nextChunk
} else {
yield { bytes: encodeTerminalStreamText(nextChunk.text), seq: nextChunk.seq }
yield {
bytes: encodeTerminalStreamText(nextChunk.text),
displayLength: nextChunk.text.length,
seq: nextChunk.seq,
sourceRanges: sliceTerminalOutputSourceRanges(
meta?.sourceRanges,
chunkStart - nextChunk.text.length,
chunkStart
)
}
}
}
chunkBytes += partBytes
@@ -124,10 +147,65 @@ export function* iterateTerminalOutputFrameChunks(
if (shouldDelayFinalSeq) {
// Why: only the final frame can safely carry the high-water mark when rawLength can't map back to UTF-16 offsets.
if (delayedChunk) {
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
yield {
bytes: encodeTerminalStreamText(delayedChunk.text),
displayLength: delayedChunk.text.length
}
}
yield {
bytes: encodeTerminalStreamText(finalChunk.text),
displayLength: finalChunk.text.length,
seq: meta.seq
}
yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: meta.seq }
return
}
yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: finalChunk.seq }
yield {
bytes: encodeTerminalStreamText(finalChunk.text),
displayLength: finalChunk.text.length,
seq: finalChunk.seq,
sourceRanges: sliceTerminalOutputSourceRanges(
meta?.sourceRanges,
data.length - finalChunk.text.length,
data.length
)
}
}
export function sliceTerminalOutputSourceRanges(
sourceRanges: readonly TerminalOutputSourceRange[] | undefined,
displayStartOffset: number,
displayEndOffset: number
): readonly TerminalOutputSourceRange[] | undefined {
if (!sourceRanges || sourceRanges.length === 0) {
return undefined
}
const baseDisplayStart = sourceRanges[0]!.displayStart
const sliceStart = baseDisplayStart + displayStartOffset
const sliceEnd = baseDisplayStart + displayEndOffset
const selected: TerminalOutputSourceRange[] = []
for (const range of sourceRanges) {
const start = Math.max(sliceStart, range.displayStart)
const end = Math.min(sliceEnd, range.displayEnd)
if (end <= start) {
continue
}
if (!range.splittable && (start !== range.displayStart || end !== range.displayEnd)) {
throw new Error('terminal_source_range_indivisible_split')
}
const sourceStartSu = range.sourceStartSu + (start - range.displayStart)
selected.push(
Object.freeze({
...range,
displayStart: start,
displayEnd: end,
sourceStartSu,
sourceEndSu: sourceStartSu + (end - start),
transform: Object.freeze({
...range.transform,
rawLengthSu: end - start
})
})
)
}
return Object.freeze(selected)
}
@@ -0,0 +1,96 @@
import { describe, expect, it } from 'vitest'
import { TERMINAL_STREAM_CHUNK_BYTES } from '../../../shared/terminal-multiplex-flow-control'
import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range'
import { iterateTerminalOutputFrameChunks } from './terminal-output-frame-chunks'
function range(overrides: Partial<TerminalOutputSourceRange>): TerminalOutputSourceRange {
return {
id: 'pty-1',
spanId: 'span-1',
providerGeneration: 2,
clientGeneration: 3,
ownerGeneration: 4,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 1,
displayStart: 0,
displayEnd: 1,
splittable: true,
transform: { transformed: false, rawLengthSu: 1, scalarSafe: true },
...overrides
}
}
describe('terminal output frame source ranges', () => {
it('maps encoded chunk boundaries to exact ordered source subranges', () => {
const data = 'a'.repeat(TERMINAL_STREAM_CHUNK_BYTES + 2)
const frames = Array.from(
iterateTerminalOutputFrameChunks(data, {
seq: data.length,
rawLength: data.length,
sourceRanges: [
range({
sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
displayEnd: TERMINAL_STREAM_CHUNK_BYTES - 1,
transform: {
transformed: false,
rawLengthSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
scalarSafe: true
}
}),
range({
spanId: 'span-2',
sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
sourceEndSu: data.length,
displayStart: TERMINAL_STREAM_CHUNK_BYTES - 1,
displayEnd: data.length,
transform: {
transformed: false,
rawLengthSu: data.length - TERMINAL_STREAM_CHUNK_BYTES + 1,
scalarSafe: true
}
})
]
})
)
expect(frames).toHaveLength(2)
expect(frames[0]?.sourceRanges).toEqual([
expect.objectContaining({
spanId: 'span-1',
sourceStartSu: 0,
sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES - 1
}),
expect.objectContaining({
spanId: 'span-2',
sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES
})
])
expect(frames[1]?.sourceRanges).toEqual([
expect.objectContaining({
spanId: 'span-2',
sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES,
sourceEndSu: data.length
})
])
})
it('keeps transformed source ranges indivisible', () => {
const sourceRange = range({
sourceEndSu: 9,
displayEnd: 3,
splittable: false,
transform: { transformed: true, rawLengthSu: 9, scalarSafe: false }
})
const [frame] = iterateTerminalOutputFrameChunks('xyz', {
seq: 9,
rawLength: 9,
transformed: true,
sourceRanges: [sourceRange]
})
expect(frame?.sourceRanges).toEqual([sourceRange])
})
})
@@ -0,0 +1,332 @@
import { describe, expect, it, vi } from 'vitest'
import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range'
import {
TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES,
TerminalSourceRangeLedger
} from './terminal-source-range-ledger'
import { TerminalSourceRangeRegistry } from './terminal-source-range-registry'
function range(overrides: Partial<TerminalOutputSourceRange> = {}): TerminalOutputSourceRange {
return {
id: 'pty-1',
spanId: 'span-1',
providerGeneration: 4,
clientGeneration: 2,
ownerGeneration: 3,
ptyIncarnation: 'incarnation-1',
deliveryToken: 'token-1',
sourceStartSu: 0,
sourceEndSu: 4,
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
...overrides
}
}
describe('TerminalSourceRangeLedger', () => {
it('advances partial byte credit without settling its covering source frame', () => {
const release = vi.fn()
const ledger = new TerminalSourceRangeLedger('generation-1', {
canReserve: () => true,
reserve: () => true,
release,
close: () => {}
})
ledger.accept(100, 100, [
range({
sourceEndSu: 100,
displayEnd: 100,
transform: { transformed: false, rawLengthSu: 100, scalarSafe: true }
})
])
expect(ledger.acknowledge('stale-generation', 40)).toEqual({
status: 'stale-generation',
settled: []
})
expect(ledger.acknowledge('generation-1', 40)).toEqual({
status: 'accepted',
acknowledgedBytes: 40,
settled: []
})
expect(ledger.getDebugSnapshot()).toMatchObject({
ackedEndByte: 40,
retainedBytes: 60,
frames: 1
})
expect(release).toHaveBeenLastCalledWith(40)
expect(ledger.acknowledge('generation-1', 100)).toMatchObject({
status: 'accepted',
acknowledgedBytes: 60,
settled: [{ spanId: 'span-1', sourceStartSu: 0, sourceEndSu: 100 }]
})
expect(ledger.getDebugSnapshot()).toMatchObject({
ackedEndByte: 100,
retainedBytes: 0,
frames: 0
})
expect(release.mock.calls).toEqual([[40], [60]])
})
it('admits a contiguous recovered delivery while the prior token remains unsettled', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
expect(ledger.accept(4, 4, [range()])).not.toBeNull()
expect(
ledger.accept(4, 4, [
range({
spanId: 'span-recovered',
clientGeneration: 3,
ownerGeneration: 4,
deliveryToken: 'token-2',
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
})
])
).not.toBeNull()
expect(ledger.acknowledge('generation-1', 4)).toMatchObject({
status: 'accepted',
settled: [{ spanId: 'span-1', deliveryToken: 'token-1' }]
})
expect(ledger.acknowledge('generation-1', 8)).toMatchObject({
status: 'accepted',
settled: [{ spanId: 'span-recovered', deliveryToken: 'token-2' }]
})
expect(
ledger.prepareAccept(4, 4, [
range({
spanId: 'span-stale',
sourceStartSu: 8,
sourceEndSu: 12,
displayStart: 8,
displayEnd: 12
})
]).status
).toBe('cross-generation')
})
it('settles only complete frames when a cumulative ACK crosses frame boundaries', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
ledger.accept(5, 4, [range()])
ledger.accept(7, 5, [
range({
spanId: 'span-2',
sourceStartSu: 4,
sourceEndSu: 9,
displayStart: 4,
displayEnd: 9,
transform: { transformed: false, rawLengthSu: 5, scalarSafe: true }
})
])
expect(ledger.acknowledge('generation-1', 3)).toEqual({
status: 'accepted',
acknowledgedBytes: 3,
settled: []
})
expect(ledger.getDebugSnapshot()).toMatchObject({
ackedEndByte: 3,
retainedBytes: 9,
frames: 2
})
expect(ledger.acknowledge('generation-1', 12)).toMatchObject({
status: 'accepted',
acknowledgedBytes: 9,
settled: [{ spanId: 'span-1' }, { spanId: 'span-2' }]
})
})
it('rejects malformed coverage, gaps, and later cross-token mappings', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
expect(ledger.accept(5, 3, [range()])).toBeNull()
expect(ledger.accept(5, 4, [range()])).not.toBeNull()
expect(
ledger.accept(5, 4, [
range({
spanId: 'gap',
sourceStartSu: 5,
sourceEndSu: 9,
displayStart: 4,
displayEnd: 8
})
])
).toBeNull()
expect(
ledger.prepareAccept(5, 4, [
range({
spanId: 'other',
deliveryToken: 'token-2',
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
})
]).status
).toBe('cross-generation')
expect(ledger.getDebugSnapshot()).toMatchObject({ retainedBytes: 5, frames: 1 })
})
it('rejects mixing mapped and unmapped output in either order', () => {
const mapped = new TerminalSourceRangeLedger('mapped')
expect(mapped.accept(5, 4, [range()])).not.toBeNull()
expect(mapped.prepareAccept(1, 1, []).status).toBe('invalid')
const unmapped = new TerminalSourceRangeLedger('unmapped')
expect(unmapped.accept(1, 1, [])).not.toBeNull()
expect(unmapped.prepareAccept(5, 4, [range()]).status).toBe('invalid')
})
it('rejects excessive, stale, notification-shaped, and late settlement', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
ledger.accept(5, 4, [range()])
expect(ledger.acknowledge('generation-2', 5).status).toBe('stale-generation')
expect(ledger.acknowledge('generation-1', 6).status).toBe('excessive')
expect(ledger.acknowledge('generation-1', Number.NaN).status).toBe('invalid')
const transfer = ledger.beginTransfer()
transfer.commit()
expect(ledger.acknowledge('generation-1', 5).status).toBe('invalid')
expect(ledger.getDebugSnapshot()).toMatchObject({ ackedEndByte: 0, closed: true })
})
it('restores all mappings when an atomic transfer rolls back', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
ledger.accept(5, 4, [range()])
const transfer = ledger.beginTransfer()
expect(transfer.frames).toHaveLength(1)
expect(ledger.canAccept(1)).toBe(false)
transfer.rollback()
expect(ledger.getDebugSnapshot()).toMatchObject({
retainedBytes: 5,
frames: 1,
transferring: false,
closed: false
})
const retry = ledger.beginTransfer()
retry.commit()
expect(ledger.getDebugSnapshot()).toMatchObject({
retainedBytes: 0,
frames: 0,
closed: true
})
})
it('replaces covered mappings without synthesizing encoded-byte credit', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
ledger.accept(5, 4, [range()], 4)
ledger.accept(
7,
4,
[
range({
spanId: 'span-trailing',
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
})
],
8
)
const replacement = ledger.planSourceRangeReplacement(8)
expect(replacement).not.toBeNull()
expect(() => replacement?.commit()).not.toThrow()
expect(ledger.acknowledge('generation-1', 12)).toMatchObject({
status: 'accepted',
acknowledgedBytes: 12,
settled: []
})
expect(
ledger.accept(
3,
2,
[
range({
spanId: 'span-live',
sourceStartSu: 8,
sourceEndSu: 10,
displayStart: 8,
displayEnd: 10,
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
})
],
10
)
).not.toBeNull()
})
it('rejects an admitted trailing mapping before authoritative commit', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
ledger.accept(5, 4, [range()], 4)
ledger.accept(
7,
4,
[
range({
spanId: 'span-trailing',
sourceStartSu: 4,
sourceEndSu: 8,
displayStart: 4,
displayEnd: 8
})
],
8
)
expect(ledger.planSourceRangeReplacement(4)).toBeNull()
})
it('rejects an unsequenced mapped frame before replacement commit', () => {
const ledger = new TerminalSourceRangeLedger('generation-1')
ledger.accept(5, 4, [range()])
expect(ledger.planSourceRangeReplacement(4)).toBeNull()
})
it('rolls back a pre-send admission without accepting a byte boundary', () => {
const registry = new TerminalSourceRangeRegistry()
const ledger = registry.open('generation-1')!
const prepared = ledger.prepareAccept(5, 4, [range()])
expect(prepared.status).toBe('ready')
if (prepared.status !== 'ready') {
throw new Error('expected source range admission')
}
expect(registry.getDebugSnapshot().retainedBytes).toBe(5)
prepared.admission.rollback()
expect(ledger.getDebugSnapshot()).toMatchObject({
acceptedEndByte: 0,
retainedBytes: 0,
frames: 0
})
expect(registry.getDebugSnapshot().retainedBytes).toBe(0)
})
it('bounds aggregate retained mapping bytes and releases them on ACK and close', () => {
const registry = new TerminalSourceRangeRegistry()
const ledgers = Array.from({ length: 9 }, (_, index) => registry.open(`stream-${index}`)!)
for (const ledger of ledgers.slice(0, 8)) {
expect(ledger.accept(TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES, 0, [])).not.toBeNull()
}
expect(ledgers[8]!.canAccept(1)).toBe(false)
expect(registry.getDebugSnapshot().retainedBytes).toBe(16 * 1024 * 1024)
expect(ledgers[0]!.acknowledge('stream-0', TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES).status).toBe(
'accepted'
)
expect(ledgers[8]!.accept(1, 0, [])).not.toBeNull()
for (const ledger of ledgers) {
ledger.close()
}
expect(registry.getDebugSnapshot()).toEqual({ streams: 0, retainedBytes: 0 })
})
})
@@ -0,0 +1,310 @@
import {
sameTerminalOutputSourceIdentity,
type TerminalOutputSourceRange
} from '../../../shared/terminal-output-source-range'
import {
canPlanTerminalSourceRangeReplacement,
freezeTerminalOutputSourceRanges,
replaceTerminalSourceRangeFrames,
type TerminalSourceRangeFrame,
validateTerminalSourceRangeFrame
} from './terminal-source-range-validation'
export type { TerminalSourceRangeFrame } from './terminal-source-range-validation'
export const TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES = 2 * 1024 * 1024
export type TerminalSourceRangeAckResult =
| {
status: 'accepted'
acknowledgedBytes: number
settled: readonly TerminalOutputSourceRange[]
}
| { status: 'duplicate'; settled: readonly [] }
| {
status: 'invalid' | 'stale-generation' | 'excessive' | 'cross-generation'
settled: readonly []
}
export type TerminalSourceRangeAdmission = Readonly<{
frame: TerminalSourceRangeFrame
commit: () => boolean
rollback: () => void
}>
export type TerminalSourceRangeAdmissionResult =
| { status: 'ready'; admission: TerminalSourceRangeAdmission }
| { status: 'capacity' | 'invalid' | 'cross-generation' }
export type TerminalSourceRangeTransfer = Readonly<{
frames: readonly TerminalSourceRangeFrame[]
commit: () => void
rollback: () => void
}>
export type TerminalSourceRangeBudget = {
canReserve: (bytes: number) => boolean
reserve: (bytes: number) => boolean
release: (bytes: number) => void
close: () => void
}
const STANDALONE_BUDGET: TerminalSourceRangeBudget = {
canReserve: () => true,
reserve: () => true,
release: () => {},
close: () => {}
}
function isRecoveredSourceIdentity(
previous: TerminalOutputSourceRange,
next: TerminalOutputSourceRange
): boolean {
return (
previous.id === next.id &&
previous.providerGeneration === next.providerGeneration &&
previous.ptyIncarnation === next.ptyIncarnation &&
next.clientGeneration > previous.clientGeneration &&
next.ownerGeneration > previous.ownerGeneration &&
next.deliveryToken !== previous.deliveryToken
)
}
export class TerminalSourceRangeLedger {
private acceptedEndByte = 0
private ackedEndByte = 0
private retainedBytes = 0
private frames: TerminalSourceRangeFrame[] = []
private mappingMode: 'mapped' | 'unmapped' | null = null
private boundRange: TerminalOutputSourceRange | null = null
private mappedSourceEndSu: number | null = null
private mappedDisplayEnd: number | null = null
private pending: TerminalSourceRangeAdmission | null = null
private transferring = false
private closed = false
constructor(
readonly streamGeneration: string,
private readonly budget: TerminalSourceRangeBudget = STANDALONE_BUDGET
) {
if (!streamGeneration) {
throw new Error('terminal_source_range_generation_required')
}
}
canAccept(encodedBytes: number): boolean {
return (
!this.closed &&
!this.transferring &&
!this.pending &&
Number.isSafeInteger(encodedBytes) &&
encodedBytes > 0 &&
this.retainedBytes + encodedBytes <= TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES &&
this.budget.canReserve(encodedBytes)
)
}
prepareAccept(
encodedBytes: number,
displayLength: number,
sourceRanges: readonly TerminalOutputSourceRange[],
outputSeq?: number
): TerminalSourceRangeAdmissionResult {
if (!this.canAccept(encodedBytes)) {
return { status: 'capacity' }
}
if (!validateTerminalSourceRangeFrame(displayLength, sourceRanges)) {
return { status: 'invalid' }
}
const nextMappingMode = sourceRanges.length > 0 ? 'mapped' : 'unmapped'
if (this.mappingMode && this.mappingMode !== nextMappingMode) {
return { status: 'invalid' }
}
const first = sourceRanges[0]
if (
first &&
(this.boundRange
? !sameTerminalOutputSourceIdentity(this.boundRange, first) &&
!isRecoveredSourceIdentity(this.boundRange, first)
: !sourceRanges.every((range) => sameTerminalOutputSourceIdentity(first, range)))
) {
return { status: 'cross-generation' }
}
if (
first &&
((this.mappedSourceEndSu !== null && first.sourceStartSu !== this.mappedSourceEndSu) ||
(this.mappedDisplayEnd !== null && first.displayStart !== this.mappedDisplayEnd))
) {
return { status: 'invalid' }
}
if (!this.budget.reserve(encodedBytes)) {
return { status: 'capacity' }
}
const ranges = freezeTerminalOutputSourceRanges(sourceRanges)
const frame = Object.freeze({
encodedStartByte: this.acceptedEndByte,
encodedEndByte: this.acceptedEndByte + encodedBytes,
displayLength,
...(typeof outputSeq === 'number' ? { outputSeq } : {}),
sourceRanges: ranges
})
let finished = false
const admission: TerminalSourceRangeAdmission = Object.freeze({
frame,
commit: () => {
if (finished || this.pending !== admission || this.closed || this.transferring) {
return false
}
finished = true
this.pending = null
this.acceptedEndByte = frame.encodedEndByte
this.retainedBytes += encodedBytes
this.frames.push(frame)
this.mappingMode ??= nextMappingMode
const last = ranges.at(-1)
if (first && last) {
this.boundRange = first
this.mappedSourceEndSu = last.sourceEndSu
this.mappedDisplayEnd = last.displayEnd
}
return true
},
rollback: () => {
if (finished) {
return
}
finished = true
if (this.pending === admission) {
this.pending = null
}
this.budget.release(encodedBytes)
}
})
this.pending = admission
return { status: 'ready', admission }
}
accept(
encodedBytes: number,
displayLength: number,
sourceRanges: readonly TerminalOutputSourceRange[],
outputSeq?: number
): TerminalSourceRangeFrame | null {
const prepared = this.prepareAccept(encodedBytes, displayLength, sourceRanges, outputSeq)
if (prepared.status !== 'ready' || !prepared.admission.commit()) {
return null
}
return prepared.admission.frame
}
acknowledge(streamGeneration: string, ackedEndByte: number): TerminalSourceRangeAckResult {
if (streamGeneration !== this.streamGeneration) {
return { status: 'stale-generation', settled: [] }
}
if (this.closed || this.transferring || this.pending) {
return { status: 'invalid', settled: [] }
}
if (
!Number.isSafeInteger(ackedEndByte) ||
ackedEndByte < 0 ||
ackedEndByte < this.ackedEndByte
) {
return { status: 'invalid', settled: [] }
}
if (ackedEndByte > this.acceptedEndByte) {
return { status: 'excessive', settled: [] }
}
if (ackedEndByte === this.ackedEndByte) {
return { status: 'duplicate', settled: [] }
}
const acknowledgedBytes = ackedEndByte - this.ackedEndByte
const settled: TerminalOutputSourceRange[] = []
let frameCount = 0
for (const frame of this.frames) {
if (frame.encodedEndByte > ackedEndByte) {
break
}
settled.push(...frame.sourceRanges)
frameCount++
}
this.ackedEndByte = ackedEndByte
this.frames.splice(0, frameCount)
this.retainedBytes -= acknowledgedBytes
this.budget.release(acknowledgedBytes)
return { status: 'accepted', acknowledgedBytes, settled: Object.freeze(settled) }
}
planSourceRangeReplacement(snapshotSeq: number): Readonly<{ commit: () => void }> | null {
const unavailable = this.closed || this.transferring || this.pending
if (unavailable || !canPlanTerminalSourceRangeReplacement(this.frames, snapshotSeq)) {
return null
}
const replacement = replaceTerminalSourceRangeFrames(this.frames, snapshotSeq)
let committed = false
return Object.freeze({
commit: () => {
if (committed || this.closed) {
return
}
committed = true
Object.assign(this, replacement)
}
})
}
beginTransfer(): TerminalSourceRangeTransfer {
if (this.closed || this.transferring || this.pending) {
throw new Error('terminal_source_range_transfer_invalid')
}
this.transferring = true
let finished = false
const frames = Object.freeze(this.frames.slice())
return Object.freeze({
frames,
commit: () => {
if (finished) {
return
}
finished = true
this.budget.release(this.retainedBytes)
this.frames = []
this.retainedBytes = 0
this.transferring = false
this.closed = true
this.budget.close()
},
rollback: () => {
if (finished) {
return
}
finished = true
this.transferring = false
}
})
}
close(): void {
if (this.closed) {
return
}
this.pending?.rollback()
this.budget.release(this.retainedBytes)
this.frames = []
this.retainedBytes = 0
this.transferring = false
this.closed = true
this.budget.close()
}
getDebugSnapshot() {
return {
acceptedEndByte: this.acceptedEndByte,
ackedEndByte: this.ackedEndByte,
retainedBytes: this.retainedBytes,
frames: this.frames.length,
transferring: this.transferring,
closed: this.closed,
bound: this.boundRange !== null
}
}
}
@@ -0,0 +1,43 @@
import { TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION } from '../../../shared/terminal-multiplex-flow-control'
import {
TerminalSourceRangeLedger,
type TerminalSourceRangeBudget
} from './terminal-source-range-ledger'
export const TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES = 16 * 1024 * 1024
export class TerminalSourceRangeRegistry {
private readonly ledgers = new Set<TerminalSourceRangeLedger>()
private retainedBytes = 0
open(streamGeneration: string): TerminalSourceRangeLedger | null {
if (this.ledgers.size >= TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION) {
return null
}
let ledger: TerminalSourceRangeLedger
const budget: TerminalSourceRangeBudget = {
canReserve: (bytes) =>
this.retainedBytes + bytes <= TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES,
reserve: (bytes) => {
if (this.retainedBytes + bytes > TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES) {
return false
}
this.retainedBytes += bytes
return true
},
release: (bytes) => {
this.retainedBytes = Math.max(0, this.retainedBytes - bytes)
},
close: () => {
this.ledgers.delete(ledger)
}
}
ledger = new TerminalSourceRangeLedger(streamGeneration, budget)
this.ledgers.add(ledger)
return ledger
}
getDebugSnapshot() {
return { streams: this.ledgers.size, retainedBytes: this.retainedBytes }
}
}
@@ -0,0 +1,99 @@
import {
assertTerminalOutputSourceRange,
sameTerminalOutputSourceIdentity,
type TerminalOutputSourceRange
} from '../../../shared/terminal-output-source-range'
export type TerminalSourceRangeFrame = Readonly<{
encodedStartByte: number
encodedEndByte: number
displayLength: number
outputSeq?: number
sourceRanges: readonly TerminalOutputSourceRange[]
}>
export function freezeTerminalOutputSourceRanges(
ranges: readonly TerminalOutputSourceRange[]
): readonly TerminalOutputSourceRange[] {
return Object.freeze(
ranges.map((range) =>
Object.freeze({
...range,
transform: Object.freeze({ ...range.transform })
})
)
)
}
export function validateTerminalSourceRangeFrame(
displayLength: number,
ranges: readonly TerminalOutputSourceRange[]
): boolean {
if (!Number.isSafeInteger(displayLength) || displayLength < 0) {
return false
}
if (ranges.length === 0) {
return true
}
try {
for (const range of ranges) {
assertTerminalOutputSourceRange(range)
}
} catch {
return false
}
const first = ranges[0]!
let previous = first
for (const range of ranges.slice(1)) {
if (
!sameTerminalOutputSourceIdentity(first, range) ||
range.sourceStartSu !== previous.sourceEndSu ||
range.displayStart !== previous.displayEnd
) {
return false
}
previous = range
}
return previous.displayEnd - first.displayStart === displayLength
}
export function replaceTerminalSourceRangeFrames(
frames: readonly TerminalSourceRangeFrame[],
snapshotSeq: number
): Readonly<{
frames: TerminalSourceRangeFrame[]
mappingMode: 'mapped' | null
boundRange: TerminalOutputSourceRange | null
mappedSourceEndSu: number | null
mappedDisplayEnd: number | null
}> {
const replaced = frames.map((frame) =>
typeof frame.outputSeq === 'number' && frame.outputSeq <= snapshotSeq
? Object.freeze({ ...frame, sourceRanges: Object.freeze([]) })
: frame
)
const remainingRanges = replaced.flatMap((frame) => frame.sourceRanges)
const last = remainingRanges.at(-1)
return Object.freeze({
frames: replaced,
mappingMode: remainingRanges.length > 0 ? 'mapped' : null,
boundRange: last ?? null,
mappedSourceEndSu: last?.sourceEndSu ?? null,
mappedDisplayEnd: last?.displayEnd ?? null
})
}
export function canPlanTerminalSourceRangeReplacement(
frames: readonly TerminalSourceRangeFrame[],
snapshotSeq: number
): boolean {
return (
Number.isSafeInteger(snapshotSeq) &&
snapshotSeq >= 0 &&
frames.every(
(frame) =>
frame.sourceRanges.length === 0 ||
(typeof frame.outputSeq === 'number' && frame.outputSeq <= snapshotSeq)
)
)
}
+264
View File
@@ -0,0 +1,264 @@
import {
containFrameDecoderContinuation,
publishFrameDecoderError,
type DecodedFrame,
type FrameDecoderOptions
} from '../../shared/relay-frame-decoder-contract'
import { RelayFrameBuffer } from '../../shared/relay-frame-buffer'
export {
FrameDecoderContinuationError,
type DecodedFrame,
type FrameDecoderOptions
} from '../../shared/relay-frame-decoder-contract'
export const HEADER_LENGTH = 13
export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024
export const FRAME_DECODER_MAX_FRAMES_PER_TURN = 64
export const FRAME_DECODER_MAX_BYTES_PER_TURN = MAX_MESSAGE_SIZE + HEADER_LENGTH
export const FRAME_DECODER_MAX_TURN_MS = 4,
FRAME_DECODER_MAX_RETAINED_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024
export class FrameDecoder {
private readonly buffer = new RelayFrameBuffer()
private oversizedPayloadBytesRemaining = 0
private onFrame: (frame: DecodedFrame) => void
private onError: ((err: Error) => void) | null
private maxFramesPerTurn: number
private maxBytesPerTurn: number
private maxTurnMs: number
private now: () => number
private schedule: (callback: () => void) => unknown
private cancelScheduled: (handle: unknown) => void
private pause: (() => void) | null
private resume: (() => void) | null
private continuationHandle: unknown
private continuationHandleAssigned = false
private continuationScheduled = false
private paused = false
private draining = false
private generation = 0
constructor(
onFrame: (frame: DecodedFrame) => void,
onError?: (err: Error) => void,
options: FrameDecoderOptions = {}
) {
this.onFrame = onFrame
this.onError = onError ?? null
this.maxFramesPerTurn = positiveLimit(
options.maxFramesPerTurn,
FRAME_DECODER_MAX_FRAMES_PER_TURN
)
this.maxBytesPerTurn = positiveLimit(options.maxBytesPerTurn, FRAME_DECODER_MAX_BYTES_PER_TURN)
this.maxTurnMs = positiveLimit(options.maxTurnMs, FRAME_DECODER_MAX_TURN_MS)
this.now = options.now ?? Date.now
this.schedule = options.schedule ?? ((callback) => setImmediate(callback))
this.cancelScheduled =
options.cancelScheduled ?? ((handle) => clearImmediate(handle as NodeJS.Immediate))
this.pause = options.pause ?? null
this.resume = options.resume ?? null
}
feed(chunk: Buffer | Uint8Array): void {
const buf = Buffer.isBuffer(chunk)
? chunk
: Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)
const retained = this.buffer.length + buf.length
if (retained > FRAME_DECODER_MAX_RETAINED_BYTES) {
this.reset()
publishFrameDecoderError(
this.onError,
new Error(`Frame decoder retained-input limit exceeded: ${retained}`)
)
return
}
if (buf.length > 0) {
this.buffer.append(buf)
}
if (!this.draining && !this.continuationScheduled) {
this.drainTurn()
}
}
reset(): void {
this.generation += 1
this.cancelContinuation()
this.buffer.clear()
this.oversizedPayloadBytesRemaining = 0
this.releasePause()
}
drain(): Buffer {
const out = this.buffer.drain()
this.reset()
return out
}
private drainTurn(): void {
if (this.draining) {
return
}
this.draining = true
const generation = this.generation
const startedAt = this.now()
let frames = 0
let bytes = 0
try {
while (generation === this.generation) {
if (
frames >= this.maxFramesPerTurn ||
bytes >= this.maxBytesPerTurn ||
(frames > 0 && this.now() - startedAt >= this.maxTurnMs)
) {
break
}
const discarded = this.discardOversizedPayload(bytes)
if (discarded > 0) {
bytes += discarded
continue
}
if (this.buffer.length < HEADER_LENGTH) {
break
}
const header = this.buffer.peek(HEADER_LENGTH)
const length = header.readUInt32BE(9)
if (length > MAX_MESSAGE_SIZE) {
this.buffer.discard(HEADER_LENGTH)
this.oversizedPayloadBytesRemaining = length
bytes += HEADER_LENGTH
publishFrameDecoderError(
this.onError,
new Error(`Frame payload too large: ${length} bytes — discarded`)
)
continue
}
const totalLength = HEADER_LENGTH + length
if (this.buffer.length < totalLength) {
break
}
if (frames > 0 && bytes + totalLength > this.maxBytesPerTurn) {
break
}
const framed = this.buffer.take(totalLength)
frames += 1
bytes += totalLength
this.onFrame({
type: framed[0],
id: framed.readUInt32BE(1),
ack: framed.readUInt32BE(5),
payload: framed.subarray(HEADER_LENGTH, totalLength)
})
}
} finally {
this.draining = false
}
if (generation !== this.generation) {
return
}
if (this.hasRunnableWork()) {
this.scheduleContinuation()
} else {
this.releasePause()
}
}
private discardOversizedPayload(bytes: number): number {
if (this.oversizedPayloadBytesRemaining === 0 || this.buffer.length === 0) {
return 0
}
const discarded = Math.min(
this.oversizedPayloadBytesRemaining,
this.buffer.length,
Math.max(1, this.maxBytesPerTurn - bytes)
)
this.buffer.discard(discarded)
this.oversizedPayloadBytesRemaining -= discarded
return discarded
}
private hasRunnableWork(): boolean {
if (this.oversizedPayloadBytesRemaining > 0) {
return this.buffer.length > 0
}
if (this.buffer.length < HEADER_LENGTH) {
return false
}
const length = this.buffer.peek(HEADER_LENGTH).readUInt32BE(9)
return length > MAX_MESSAGE_SIZE || this.buffer.length >= HEADER_LENGTH + length
}
private scheduleContinuation(): void {
if (this.continuationScheduled) {
return
}
const generation = this.generation
this.continuationScheduled = true
try {
this.acquirePause()
} catch (error) {
this.continuationScheduled = false
throw error
}
if (generation !== this.generation) {
this.continuationScheduled = false
return
}
try {
this.continuationHandle = this.schedule(() => {
if (!this.continuationScheduled || generation !== this.generation) {
return
}
this.continuationScheduled = false
this.continuationHandleAssigned = false
this.continuationHandle = undefined
try {
this.drainTurn()
} catch (error) {
containFrameDecoderContinuation(() => this.reset(), this.onError, error)
}
})
this.continuationHandleAssigned = true
} catch (error) {
this.continuationScheduled = false
this.continuationHandle = undefined
this.releasePause()
throw error
}
}
private cancelContinuation(): void {
if (!this.continuationScheduled) {
return
}
this.continuationScheduled = false
if (this.continuationHandleAssigned) {
this.cancelScheduled(this.continuationHandle)
}
this.continuationHandleAssigned = false
this.continuationHandle = undefined
}
private acquirePause(): void {
if (!this.paused) {
this.paused = true
try {
this.pause?.()
} catch (error) {
this.paused = false
throw error
}
}
}
private releasePause(): void {
if (this.paused) {
this.paused = false
this.resume?.()
}
}
}
const positiveLimit = (value: number | undefined, fallback: number): number =>
value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback
@@ -0,0 +1,263 @@
import { describe, expect, it, vi } from 'vitest'
import {
FrameDecoder,
FrameDecoderContinuationError,
FRAME_DECODER_MAX_RETAINED_BYTES,
HEADER_LENGTH,
MAX_MESSAGE_SIZE,
MessageType,
encodeFrame,
type DecodedFrame
} from './relay-protocol'
function createScheduler(): {
schedule: (callback: () => void) => number
cancel: (handle: unknown) => void
runNext: () => void
pending: () => number
} {
let nextHandle = 1
const callbacks = new Map<number, () => void>()
return {
schedule: (callback) => {
const handle = nextHandle++
callbacks.set(handle, callback)
return handle
},
cancel: (handle) => callbacks.delete(handle as number),
runNext: () => {
const entry = callbacks.entries().next().value as [number, () => void] | undefined
if (!entry) {
throw new Error('No decoder continuation scheduled')
}
callbacks.delete(entry[0])
entry[1]()
},
pending: () => callbacks.size
}
}
function frame(id: number, payload = `${id}`): Buffer {
return encodeFrame(MessageType.Regular, id, 0, Buffer.from(payload))
}
describe('FrameDecoder bounded turns', () => {
it('retains at most one maximum frame plus one MiB of partial input', () => {
expect(FRAME_DECODER_MAX_RETAINED_BYTES).toBe(MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024)
const maximumFrame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(MAX_MESSAGE_SIZE))
const acceptedError = vi.fn()
const accepted = new FrameDecoder(vi.fn(), acceptedError)
accepted.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024)]))
expect(acceptedError).not.toHaveBeenCalled()
expect(accepted.drain()).toHaveLength(1024 * 1024)
const excessError = vi.fn()
const excess = new FrameDecoder(vi.fn(), excessError)
excess.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024 + 1)]))
expect(excessError).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining('retained-input') })
)
expect(excess.drain()).toHaveLength(0)
})
it('fails closed when one delivered chunk exceeds retained-input capacity', () => {
const onError = vi.fn()
const decoder = new FrameDecoder(vi.fn(), onError)
decoder.feed(Buffer.alloc(FRAME_DECODER_MAX_RETAINED_BYTES + 1))
expect(onError).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining('retained-input') })
)
expect(decoder.drain()).toHaveLength(0)
})
it('emits the first frame synchronously and preserves order through self-pause', () => {
const scheduler = createScheduler()
const seen: number[] = []
let decoder: FrameDecoder
const pause = vi.fn(() => decoder.feed(frame(4)))
const resume = vi.fn()
decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, {
maxFramesPerTurn: 1,
schedule: scheduler.schedule,
cancelScheduled: scheduler.cancel,
pause,
resume
})
decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)]))
expect(seen).toEqual([1])
expect(pause).toHaveBeenCalledTimes(1)
expect(scheduler.pending()).toBe(1)
scheduler.runNext()
scheduler.runNext()
scheduler.runNext()
expect(seen).toEqual([1, 2, 3, 4])
expect(pause).toHaveBeenCalledTimes(1)
expect(resume).toHaveBeenCalledTimes(1)
expect(scheduler.pending()).toBe(0)
})
it('bounds decoded bytes and time independently from the frame count', () => {
const byteScheduler = createScheduler()
const byteSeen: number[] = []
const first = frame(1, 'one')
const second = frame(2, 'two')
const byteDecoder = new FrameDecoder((decoded) => byteSeen.push(decoded.id), undefined, {
maxFramesPerTurn: 64,
maxBytesPerTurn: first.length,
schedule: byteScheduler.schedule,
cancelScheduled: byteScheduler.cancel
})
byteDecoder.feed(Buffer.concat([first, second]))
expect(byteSeen).toEqual([1])
byteScheduler.runNext()
expect(byteSeen).toEqual([1, 2])
const timeScheduler = createScheduler()
const timeSeen: number[] = []
let nowCalls = 0
const timeDecoder = new FrameDecoder((decoded) => timeSeen.push(decoded.id), undefined, {
maxFramesPerTurn: 64,
maxBytesPerTurn: MAX_MESSAGE_SIZE + HEADER_LENGTH,
maxTurnMs: 4,
now: () => (nowCalls++ === 0 ? 0 : 5),
schedule: timeScheduler.schedule,
cancelScheduled: timeScheduler.cancel
})
timeDecoder.feed(Buffer.concat([frame(3), frame(4)]))
expect(timeSeen).toEqual([3])
timeScheduler.runNext()
expect(timeSeen).toEqual([3, 4])
})
it('releases its pause epoch when continuation scheduling throws', () => {
const pause = vi.fn()
const resume = vi.fn()
const decoder = new FrameDecoder(() => {}, undefined, {
maxFramesPerTurn: 1,
pause,
resume,
schedule: () => {
throw new Error('scheduler unavailable')
}
})
expect(() => decoder.feed(Buffer.concat([frame(1), frame(2)]))).toThrow('scheduler unavailable')
expect(pause).toHaveBeenCalledTimes(1)
expect(resume).toHaveBeenCalledTimes(1)
decoder.reset()
expect(resume).toHaveBeenCalledTimes(1)
})
it('contains a throwing continuation, resets residue, and reports one typed error', () => {
const scheduler = createScheduler()
const seen: number[] = []
const onError = vi.fn()
const pause = vi.fn()
const resume = vi.fn()
const decoder = new FrameDecoder(
(decoded) => {
if (decoded.id === 2) {
throw new Error('frame owner failed')
}
seen.push(decoded.id)
},
onError,
{
maxFramesPerTurn: 1,
schedule: scheduler.schedule,
cancelScheduled: scheduler.cancel,
pause,
resume
}
)
decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)]))
expect(() => scheduler.runNext()).not.toThrow()
expect(seen).toEqual([1])
expect(onError).toHaveBeenCalledExactlyOnceWith(expect.any(FrameDecoderContinuationError))
expect(onError.mock.calls[0]?.[0]).toMatchObject({
name: 'FrameDecoderContinuationError',
cause: expect.objectContaining({ message: 'frame owner failed' })
})
expect(pause).toHaveBeenCalledTimes(1)
expect(resume).toHaveBeenCalledTimes(1)
expect(scheduler.pending()).toBe(0)
expect(decoder.drain()).toHaveLength(0)
decoder.feed(frame(4))
expect(seen).toEqual([1, 4])
})
it('keeps reads active for partial frames and incrementally discards oversized payloads', () => {
const errors: Error[] = []
const seen: DecodedFrame[] = []
const pause = vi.fn()
const decoder = new FrameDecoder(
(decoded) => seen.push(decoded),
(error) => errors.push(error),
{ pause }
)
const valid = frame(2, 'complete')
decoder.feed(valid.subarray(0, HEADER_LENGTH + 2))
expect(seen).toHaveLength(0)
expect(pause).not.toHaveBeenCalled()
decoder.feed(valid.subarray(HEADER_LENGTH + 2))
expect(seen.map(({ id }) => id)).toEqual([2])
const oversizedHeader = Buffer.alloc(HEADER_LENGTH)
oversizedHeader[0] = MessageType.Regular
oversizedHeader.writeUInt32BE(3, 1)
oversizedHeader.writeUInt32BE(MAX_MESSAGE_SIZE + 1, 9)
decoder.feed(Buffer.concat([oversizedHeader, Buffer.alloc(32)]))
expect(errors).toHaveLength(1)
expect(pause).not.toHaveBeenCalled()
decoder.reset()
decoder.feed(frame(4, 'after-reset'))
expect(seen.map(({ id }) => id)).toEqual([2, 4])
})
it('drain and reset cancel continuation ownership without replaying residue', () => {
const scheduler = createScheduler()
const seen: number[] = []
const resume = vi.fn()
const cancel = vi.fn(scheduler.cancel)
const decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, {
maxFramesPerTurn: 1,
schedule: scheduler.schedule,
cancelScheduled: cancel,
resume
})
const second = frame(2, 'residue')
decoder.feed(Buffer.concat([frame(1), second]))
const residue = decoder.drain()
expect(seen).toEqual([1])
expect(residue.equals(second)).toBe(true)
expect(cancel).toHaveBeenCalledTimes(1)
expect(resume).toHaveBeenCalledTimes(1)
expect(scheduler.pending()).toBe(0)
decoder.feed(Buffer.concat([frame(3), frame(4)]))
decoder.reset()
expect(cancel).toHaveBeenCalledTimes(2)
expect(scheduler.pending()).toBe(0)
decoder.feed(frame(5))
expect(seen).toEqual([1, 3, 5])
})
})
+22 -147
View File
@@ -3,17 +3,34 @@
// See design-ssh-support.md § JSON-RPC Protocol Specification.
import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../../shared/ssh-types'
import {
FrameDecoder,
FrameDecoderContinuationError,
HEADER_LENGTH,
MAX_MESSAGE_SIZE,
FRAME_DECODER_MAX_FRAMES_PER_TURN,
FRAME_DECODER_MAX_BYTES_PER_TURN,
FRAME_DECODER_MAX_TURN_MS,
FRAME_DECODER_MAX_RETAINED_BYTES
} from './relay-frame-decoder'
export {
FrameDecoder,
FrameDecoderContinuationError,
HEADER_LENGTH,
MAX_MESSAGE_SIZE,
FRAME_DECODER_MAX_FRAMES_PER_TURN,
FRAME_DECODER_MAX_BYTES_PER_TURN,
FRAME_DECODER_MAX_TURN_MS,
FRAME_DECODER_MAX_RETAINED_BYTES
}
export type { DecodedFrame, FrameDecoderOptions } from './relay-frame-decoder'
export const RELAY_VERSION = '0.1.0'
export const RELAY_SENTINEL = `ORCA-RELAY v${RELAY_VERSION} READY\n`
export const RELAY_SENTINEL_TIMEOUT_MS = 10_000
export const RELAY_REMOTE_DIR = '.orca-remote'
// ── Framing constants (VS Code ProtocolConstants) ───────────────────
export const HEADER_LENGTH = 13
export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 // 16 MB
/** Message type byte. */
export const MessageType = {
Regular: 1,
@@ -155,148 +172,6 @@ export function encodeKeepAliveFrame(id: number, ack: number): Buffer {
return encodeFrame(MessageType.KeepAlive, id, ack, Buffer.alloc(0))
}
export type DecodedFrame = {
type: number
id: number
ack: number
payload: Buffer
}
/**
* Incremental frame parser. Feed it chunks of data; it emits complete frames.
*/
export class FrameDecoder {
// Why: feed() runs on the Electron main thread for every SSH channel data
// event. Rebuilding one contiguous buffer per feed (Buffer.concat) re-copies
// every already-buffered byte for each incoming ~32KB TCP chunk — O(n²) per
// large frame (a 340KB fs.streamChunk frame cost ~2MB of memcpy). A chunk
// list assembles each frame exactly once instead.
private chunks: Buffer[] = []
private bufferedLength = 0
private onFrame: (frame: DecodedFrame) => void
private onError: ((err: Error) => void) | null
constructor(onFrame: (frame: DecodedFrame) => void, onError?: (err: Error) => void) {
this.onFrame = onFrame
this.onError = onError ?? null
}
feed(chunk: Buffer | Uint8Array): void {
const buf = Buffer.isBuffer(chunk)
? chunk
: Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)
if (buf.length > 0) {
this.chunks.push(buf)
this.bufferedLength += buf.length
}
while (this.bufferedLength >= HEADER_LENGTH) {
const header = this.peekBytes(HEADER_LENGTH)
const length = header.readUInt32BE(9)
const totalLength = HEADER_LENGTH + length
if (this.bufferedLength < totalLength) {
// Not fully received yet (also holds oversized frames until they can
// be skipped whole, keeping the decoder synchronized).
break
}
// Why: throwing here would leave the buffer in a partially consumed
// state — subsequent feed() calls would try to parse leftover payload
// bytes as a new header, corrupting every future frame. Instead we
// skip the entire oversized frame so the decoder stays synchronized.
if (length > MAX_MESSAGE_SIZE) {
this.discardBytes(totalLength)
const err = new Error(`Frame payload too large: ${length} bytes — discarded`)
if (this.onError) {
this.onError(err)
}
continue
}
const framed = this.takeBytes(totalLength)
const frame: DecodedFrame = {
type: framed[0],
id: framed.readUInt32BE(1),
ack: framed.readUInt32BE(5),
payload: framed.subarray(HEADER_LENGTH, totalLength)
}
this.onFrame(frame)
}
}
reset(): void {
this.chunks = []
this.bufferedLength = 0
}
/** View of the first `count` buffered bytes without consuming them. */
private peekBytes(count: number): Buffer {
const first = this.chunks[0]
if (first.length >= count) {
return first
}
const out = Buffer.allocUnsafe(count)
let copied = 0
for (const part of this.chunks) {
copied += part.copy(out, copied, 0, Math.min(part.length, count - copied))
if (copied >= count) {
break
}
}
return out
}
/** Consume and return the first `count` buffered bytes (single copy). */
private takeBytes(count: number): Buffer {
const first = this.chunks[0]
if (first.length === count) {
this.chunks.shift()
this.bufferedLength -= count
return first
}
if (first.length > count) {
this.chunks[0] = first.subarray(count)
this.bufferedLength -= count
return first.subarray(0, count)
}
const out = Buffer.allocUnsafe(count)
let copied = 0
while (copied < count) {
const part = this.chunks[0]
const take = Math.min(part.length, count - copied)
part.copy(out, copied, 0, take)
copied += take
if (take === part.length) {
this.chunks.shift()
} else {
this.chunks[0] = part.subarray(take)
}
}
this.bufferedLength -= count
return out
}
/** Consume the first `count` buffered bytes without assembling them. */
private discardBytes(count: number): void {
let remaining = count
while (remaining > 0) {
const part = this.chunks[0]
if (part.length <= remaining) {
this.chunks.shift()
remaining -= part.length
} else {
this.chunks[0] = part.subarray(remaining)
remaining = 0
}
}
this.bufferedLength -= count
}
}
/**
* Parse a JSON-RPC message from a frame payload.
*/
export function parseJsonRpcMessage(payload: Buffer): JsonRpcMessage {
const text = payload.toString('utf-8')
const msg = JSON.parse(text) as JsonRpcMessage
@@ -0,0 +1,203 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer'
import { encodeFrame, encodeKeepAliveFrame, HEADER_LENGTH, MessageType } from './relay-protocol'
type MockTransport = MultiplexerTransport & {
data: (chunk: Buffer) => void
written: Buffer[]
pauseReads: ReturnType<typeof vi.fn>
resumeReads: ReturnType<typeof vi.fn>
}
type MuxInternals = {
nextOutgoingSeq: number
highestAckedBySelf: number
lastReceivedAt: number
decoderReadPaused: boolean
unackedTimestamps: Map<number, number>
}
function createTransport(): MockTransport {
let onData: (chunk: Buffer) => void = () => {}
const written: Buffer[] = []
const pauseReads = vi.fn()
const resumeReads = vi.fn()
return {
write: (data) => {
written.push(data)
},
onData: (callback) => {
onData = callback
},
onClose: () => {},
pauseReads,
resumeReads,
data: (chunk) => onData(chunk),
written
}
}
function internals(mux: SshChannelMultiplexer): MuxInternals {
return mux as unknown as MuxInternals
}
describe('SshChannelMultiplexer backpressure hardening', () => {
let transport: MockTransport
let mux: SshChannelMultiplexer
beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(0)
transport = createTransport()
mux = new SshChannelMultiplexer(transport)
})
afterEach(() => {
mux.dispose()
vi.restoreAllMocks()
vi.useRealTimers()
})
it('clamps ack=0xffffffff and deletes only retained sequence keys', () => {
mux.notify('one')
mux.notify('two')
const state = internals(mux)
const deleteSpy = vi.spyOn(state.unackedTimestamps, 'delete')
transport.data(encodeKeepAliveFrame(1, 0xffffffff))
expect(deleteSpy).toHaveBeenCalledTimes(2)
expect(state.unackedTimestamps.size).toBe(0)
expect(state.highestAckedBySelf).toBe(state.nextOutgoingSeq - 1)
})
it('caps timestamps while preserving a reserved liveness entry', () => {
for (let i = 0; i < 5000; i += 1) {
mux.notify('bounded')
}
const state = internals(mux)
expect(state.unackedTimestamps.size).toBe(4095)
expect(state.unackedTimestamps.has(1)).toBe(true)
void mux.probeLiveness(60_000)
expect(state.unackedTimestamps.size).toBe(4096)
void mux.probeLiveness(60_000)
expect(state.unackedTimestamps.size).toBe(4096)
})
it('suppresses timeout while self-paused and rebases both clocks on resume', () => {
mux.dispose()
const continuations: (() => void)[] = []
vi.spyOn(globalThis, 'setImmediate').mockImplementation(
(callback: (...args: never[]) => void) => {
continuations.push(callback)
return {} as NodeJS.Immediate
}
)
transport = createTransport()
mux = new SshChannelMultiplexer(transport)
mux.notify('tracked')
const incoming = Buffer.concat(
Array.from({ length: 65 }, (_, index) => encodeKeepAliveFrame(index + 1, 0))
)
transport.data(incoming)
const state = internals(mux)
expect(state.decoderReadPaused).toBe(true)
expect(transport.pauseReads).toHaveBeenCalledTimes(1)
expect(continuations).toHaveLength(1)
vi.advanceTimersByTime(25_000)
expect(mux.isDisposed()).toBe(false)
continuations.shift()!()
const resumedAt = Date.now()
expect(state.decoderReadPaused).toBe(false)
expect(transport.resumeReads).toHaveBeenCalledTimes(1)
expect(state.lastReceivedAt).toBe(resumedAt)
expect(new Set(state.unackedTimestamps.values())).toEqual(new Set([resumedAt]))
})
it('keeps frame dispatch ordered across a decoder continuation', () => {
const seen: number[] = []
mux.onNotification((_method, params) => seen.push(params.index as number))
const frames = Array.from({ length: 65 }, (_, index) => {
const payload = Buffer.from(
JSON.stringify({ jsonrpc: '2.0', method: 'ordered', params: { index } })
)
return encodeFrame(MessageType.Regular, index + 1, 0, payload)
})
transport.data(Buffer.concat(frames))
expect(seen).toEqual(Array.from({ length: 64 }, (_, index) => index))
vi.runAllTicks()
vi.advanceTimersByTime(0)
expect(seen).toEqual(Array.from({ length: 65 }, (_, index) => index))
})
it('prioritizes source ACK and control while preserving FIFO and ordinary progress', async () => {
mux.dispose()
const written: Buffer[] = []
let drain = (): void => {}
let deliver = (_data: Buffer): void => {}
transport = {
write: (data) => {
written.push(data)
return written.length !== 1
},
onDrain: (callback) => {
drain = callback
},
onData: (callback) => {
deliver = callback
},
onClose: vi.fn(),
pauseReads: vi.fn<() => void>(),
resumeReads: vi.fn<() => void>(),
data: vi.fn<(chunk: Buffer) => void>(),
written,
close: vi.fn()
}
mux = new SshChannelMultiplexer(transport)
mux.onRequest('client.control', () => ({ accepted: true }))
const controller = new AbortController()
mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-1' })
mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-2' })
mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-3' })
mux.notify('pty.ackData', { acknowledgements: [] })
const request = mux.request('fs.scan', {}, { signal: controller.signal })
controller.abort()
mux.notify('pty.exit', { id: 'pty-1', code: 0 })
const remoteRequest = Buffer.from(
JSON.stringify({ jsonrpc: '2.0', id: 91, method: 'client.control' })
)
deliver(encodeFrame(MessageType.Regular, 1, 0, remoteRequest))
await Promise.resolve()
await Promise.resolve()
expect(written).toHaveLength(1)
drain()
const payloads = written.map((frame) =>
JSON.parse(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + frame.readUInt32BE(9)).toString())
)
expect(
payloads.map((payload) =>
payload.method === 'pty.data'
? `pty.data:${payload.params.data}`
: (payload.method ?? `response:${payload.id}`)
)
).toEqual([
'pty.data:ordinary-1',
'pty.ackData',
'fs.scan',
'rpc.cancel',
'pty.exit',
'pty.data:ordinary-2',
'response:91',
'pty.data:ordinary-3'
])
await expect(request).rejects.toMatchObject({ name: 'AbortError' })
})
})
@@ -0,0 +1,93 @@
import { describe, expect, it, vi } from 'vitest'
import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer'
function transportHarness(): {
transport: MultiplexerTransport
settlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[]
} {
const settlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[] = []
return {
transport: {
write: (_data, onSettled) => {
if (onSettled) {
settlements.push(onSettled)
}
},
supportsWriteSettlement: true,
onData: vi.fn(),
onClose: vi.fn()
},
settlements
}
}
describe('SshChannelMultiplexer notification settlement', () => {
it('reports publication only from the transport write callback', () => {
const harness = transportHarness()
const mux = new SshChannelMultiplexer(harness.transport)
const settled = vi.fn()
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
expect(settled).not.toHaveBeenCalled()
harness.settlements[0]({ ok: true })
expect(settled).toHaveBeenCalledWith({ ok: true })
mux.dispose()
})
it('reports a synchronous write failure without publishing success', () => {
const error = new Error('write failed')
const mux = new SshChannelMultiplexer({
write: () => {
throw error
},
supportsWriteSettlement: true,
onData: vi.fn(),
onClose: vi.fn()
})
const settled = vi.fn()
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
expect(settled).toHaveBeenCalledWith({ ok: false, error })
expect(mux.isDisposed()).toBe(true)
})
it('settles once when a hostile transport invokes its callback and then throws', () => {
const mux = new SshChannelMultiplexer({
write: (_data, onSettled) => {
onSettled?.({ ok: true })
throw new Error('late throw')
},
supportsWriteSettlement: true,
onData: vi.fn(),
onClose: vi.fn()
})
const settled = vi.fn()
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
expect(settled).toHaveBeenCalledOnce()
expect(settled).toHaveBeenCalledWith({ ok: true })
})
it('fails an unsettled publication when the multiplexer is disposed', () => {
const close = vi.fn()
const mux = new SshChannelMultiplexer({
write: () => false,
supportsWriteSettlement: true,
onDrain: vi.fn(),
onData: vi.fn(),
onClose: vi.fn(),
close
})
const settled = vi.fn()
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
expect(settled).not.toHaveBeenCalled()
mux.dispose()
expect(settled).toHaveBeenCalledWith({
ok: false,
error: expect.objectContaining({ code: 'DISPOSED' })
})
expect(close).toHaveBeenCalledOnce()
})
})
+63 -1
View File
@@ -12,7 +12,9 @@ function createMockTransport(): MultiplexerTransport & {
const written: Buffer[] = []
return {
write: (data: Buffer) => written.push(data),
write: (data: Buffer) => {
written.push(data)
},
onData: (cb) => dataCallbacks.push(cb),
onClose: (cb) => closeCallbacks.push(cb),
dataCallbacks,
@@ -69,6 +71,7 @@ type MuxInternals = {
disposeHandlers: unknown[]
lastReceivedAt: number
unackedTimestamps: Map<number, number>
writerSaturated: boolean
}
function getMuxInternals(instance: SshChannelMultiplexer): MuxInternals {
@@ -123,6 +126,28 @@ describe('SshChannelMultiplexer', () => {
await expect(promise).rejects.toThrow('PTY allocation failed')
})
it('runs beforeResolve before an adjacent notification in the same decoder turn', async () => {
const order: string[] = []
mux.onNotification(() => order.push('notification'))
const promise = mux.request(
'pty.attach',
{ id: 'pty-1' },
{
beforeResolve: () => order.push('beforeResolve')
}
)
transport.dataCallbacks[0](
Buffer.concat([
makeResponseFrame(1, { incarnationId: 'incarnation-1' }, 1),
makeNotificationFrame('pty.data', { id: 'pty-1', data: 'first' }, 2)
])
)
expect(order).toEqual(['beforeResolve', 'notification'])
await expect(promise).resolves.toEqual({ incarnationId: 'incarnation-1' })
})
it('times out after 30s with no response', async () => {
const promise = mux.request('pty.spawn')
@@ -328,6 +353,43 @@ describe('SshChannelMultiplexer', () => {
vi.advanceTimersByTime(25_000)
expect(mux.isDisposed()).toBe(true)
})
it('suppresses false death while locally saturated and rebases both clocks on drain', () => {
mux.dispose()
let drain = (): void => {}
const written: Buffer[] = []
const saturatedTransport: MultiplexerTransport = {
write: (data) => {
written.push(data)
return false
},
supportsWriteSettlement: true,
onDrain: (callback) => {
drain = callback
},
onData: vi.fn(),
onClose: vi.fn()
}
mux = new SshChannelMultiplexer(saturatedTransport)
vi.advanceTimersByTime(5_000)
expect(getMuxInternals(mux).writerSaturated).toBe(true)
vi.advanceTimersByTime(25_000)
expect(mux.isDisposed()).toBe(false)
expect(written).toHaveLength(1)
drain()
const resumedAt = Date.now()
const internals = getMuxInternals(mux)
expect(internals.writerSaturated).toBe(false)
expect(internals.lastReceivedAt).toBe(resumedAt)
expect(new Set(internals.unackedTimestamps.values())).toEqual(new Set([resumedAt]))
vi.advanceTimersByTime(20_000)
expect(mux.isDisposed()).toBe(false)
vi.advanceTimersByTime(5_000)
expect(mux.isDisposed()).toBe(true)
})
})
describe('wake guard (timer pause across system sleep, #7773)', () => {
+140 -38
View File
@@ -14,26 +14,36 @@ import {
type JsonRpcResponse,
type JsonRpcNotification
} from './relay-protocol'
import {
SshMultiplexerTransportWriter,
type MultiplexerTransport,
type MultiplexerWriteSettlement,
type MultiplexerWriterLane
} from './ssh-multiplexer-transport-writer'
export type MultiplexerTransport = {
write: (data: Buffer) => void
onData: (cb: (data: Buffer) => void) => void
onClose: (cb: () => void) => void
close?: () => void
}
export type { MultiplexerTransport, MultiplexerWriteSettlement }
type PendingRequest = {
resolve: (result: unknown) => void
reject: (error: Error) => void
beforeResolve?: (result: unknown) => void
timer: ReturnType<typeof setTimeout>
cleanup: () => void
}
export type SshMultiplexerRequestOptions = {
signal?: AbortSignal
timeoutMs?: number
beforeResolve?: (result: unknown) => void
}
export type NotificationHandler = (method: string, params: Record<string, unknown>) => void
export type MethodNotificationHandler = (params: Record<string, unknown>) => void
export type RequestHandler = (params: Record<string, unknown>) => Promise<unknown> | unknown
const REQUEST_TIMEOUT_MS = 30_000
const MAX_ORDINARY_UNACKED_TIMESTAMPS = 4095
const MAX_UNACKED_TIMESTAMPS = MAX_ORDINARY_UNACKED_TIMESTAMPS + 1
// Why: a tick gap far beyond the interval means the process was paused
// (system sleep, App Nap timer throttling) — not that the link is dead (#7773).
const WAKE_GAP_MS = KEEPALIVE_SEND_MS * 3
@@ -41,6 +51,7 @@ const WAKE_GAP_MS = KEEPALIVE_SEND_MS * 3
export class SshChannelMultiplexer {
private decoder: FrameDecoder
private transport: MultiplexerTransport
private writer: SshMultiplexerTransportWriter
private nextRequestId = 1
private nextOutgoingSeq = 1
private highestReceivedSeq = 0
@@ -56,6 +67,8 @@ export class SshChannelMultiplexer {
private disposeHandlers: ((reason: 'shutdown' | 'connection_lost') => void)[] = []
private connectionHealthTimer: ReturnType<typeof setInterval> | null = null
private disposed = false
private decoderReadPaused = false
private writerSaturated = false
// Track the oldest unacked outgoing message timestamp
private unackedTimestamps = new Map<number, number>()
@@ -66,10 +79,19 @@ export class SshChannelMultiplexer {
constructor(transport: MultiplexerTransport) {
this.transport = transport
this.writer = new SshMultiplexerTransportWriter(
transport,
(error) => this.handleProtocolError(error),
(saturated) => this.handleWriterSaturationChange(saturated)
)
this.decoder = new FrameDecoder(
(frame) => this.handleFrame(frame),
(err) => this.handleProtocolError(err)
(err) => this.handleProtocolError(err),
{
pause: () => this.pauseDecoderReads(),
resume: () => this.resumeDecoderReads()
}
)
transport.onData((data) => {
@@ -158,7 +180,7 @@ export class SshChannelMultiplexer {
async request(
method: string,
params?: Record<string, unknown>,
options?: { signal?: AbortSignal; timeoutMs?: number }
options?: SshMultiplexerRequestOptions
): Promise<unknown> {
if (this.disposed) {
throw new Error('Multiplexer disposed')
@@ -215,7 +237,13 @@ export class SshChannelMultiplexer {
if (options?.signal) {
options.signal.addEventListener('abort', onAbort, { once: true })
}
this.pendingRequests.set(id, { resolve, reject, timer, cleanup })
this.pendingRequests.set(id, {
resolve,
reject,
beforeResolve: options?.beforeResolve,
timer,
cleanup
})
this.sendMessage(msg)
})
}
@@ -237,6 +265,25 @@ export class SshChannelMultiplexer {
this.sendMessage(msg)
}
notifyWithSettlement(
method: string,
params: Record<string, unknown> | undefined,
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
): void {
if (this.disposed) {
onSettled({ ok: false, error: new Error('Multiplexer disposed') })
return
}
this.sendMessage(
{
jsonrpc: '2.0',
method,
...(params !== undefined ? { params } : {})
},
onSettled
)
}
/**
* Send a fresh keepalive and resolve true when any frame arrives before the
* timeout. Used on system resume to distinguish a link that survived sleep
@@ -297,6 +344,9 @@ export class SshChannelMultiplexer {
this.pendingRequests.delete(id)
}
const writerError = new Error(errorMessage) as Error & { code: string }
writerError.code = errorCode
this.writer.dispose(writerError)
this.unackedTimestamps.clear()
// Why: relay teardown can race with late provider registration; disposed
// muxes must not retain provider/session closures through subscribers.
@@ -321,34 +371,27 @@ export class SshChannelMultiplexer {
// ── Private ───────────────────────────────────────────────────────
private sendMessage(msg: JsonRpcMessage): void {
private sendMessage(
msg: JsonRpcMessage,
onSettled?: (result: MultiplexerWriteSettlement) => void
): void {
const seq = this.nextOutgoingSeq++
const frame = encodeJsonRpcFrame(msg, seq, this.highestReceivedSeq)
this.unackedTimestamps.set(seq, Date.now())
try {
this.transport.write(frame)
} catch (err) {
// Why: a remote reboot can make the SSH channel's stdin throw EPIPE
// from a timer/request path. Scope it to this mux instead of letting
// the Electron main process treat it as an uncaught exception.
this.handleProtocolError(err)
}
this.trackOutgoingTimestamp(seq, false)
this.writer.enqueue(frame, messageLane(msg), onSettled)
}
private sendKeepAlive(): void {
if (this.disposed) {
return
}
const seq = this.nextOutgoingSeq++
const seq = this.nextOutgoingSeq
const frame = encodeKeepAliveFrame(seq, this.highestReceivedSeq)
this.unackedTimestamps.set(seq, Date.now())
try {
this.transport.write(frame)
} catch (err) {
// Why: keepalive runs on an interval; without catching transport
// write failures here, a dead SSH host can terminate the whole app.
this.handleProtocolError(err)
if (!this.writer.enqueue(frame, 'liveness')) {
return
}
this.nextOutgoingSeq++
this.trackOutgoingTimestamp(seq, true)
}
private handleFrame(frame: DecodedFrame): void {
@@ -363,12 +406,16 @@ export class SshChannelMultiplexer {
this.highestReceivedSeq = frame.id
}
// Process ack from remote: discard timestamps for acked messages
if (frame.ack > this.highestAckedBySelf) {
for (let i = this.highestAckedBySelf + 1; i <= frame.ack; i++) {
this.unackedTimestamps.delete(i)
// Header ACKs are untrusted uint32 values; work stays proportional to the
// bounded set of sequence keys we actually retained.
const acknowledgedSeq = Math.min(frame.ack, this.nextOutgoingSeq - 1)
if (acknowledgedSeq > this.highestAckedBySelf) {
for (const seq of this.unackedTimestamps.keys()) {
if (seq <= acknowledgedSeq) {
this.unackedTimestamps.delete(seq)
}
}
this.highestAckedBySelf = frame.ack
this.highestAckedBySelf = acknowledgedSeq
}
if (frame.type === MessageType.KeepAlive) {
@@ -440,7 +487,12 @@ export class SshChannelMultiplexer {
Object.defineProperty(err, 'data', { value: msg.error.data })
pending.reject(err)
} else {
pending.resolve(msg.result)
try {
pending.beforeResolve?.(msg.result)
pending.resolve(msg.result)
} catch (error) {
pending.reject(error instanceof Error ? error : new Error(String(error)))
}
}
}
@@ -495,15 +547,12 @@ export class SshChannelMultiplexer {
// before this tick's fresh probe, then allow the next full window.
const resumedAfterWake = sinceLastTick > WAKE_GAP_MS
if (resumedAfterWake) {
this.lastReceivedAt = now
for (const seq of this.unackedTimestamps.keys()) {
this.unackedTimestamps.set(seq, now)
}
this.rebaseHealthClocks(now)
}
this.sendKeepAlive()
if (this.disposed || resumedAfterWake) {
if (this.disposed || resumedAfterWake || this.decoderReadPaused || this.writerSaturated) {
return
}
@@ -529,4 +578,57 @@ export class SshChannelMultiplexer {
console.warn(`[ssh-mux] Protocol error: ${err instanceof Error ? err.message : String(err)}`)
this.dispose('connection_lost')
}
private trackOutgoingTimestamp(seq: number, liveness: boolean): void {
const limit = liveness ? MAX_UNACKED_TIMESTAMPS : MAX_ORDINARY_UNACKED_TIMESTAMPS
if (this.unackedTimestamps.size < limit) {
this.unackedTimestamps.set(seq, Date.now())
}
}
private pauseDecoderReads(): void {
if (this.disposed || this.decoderReadPaused) {
return
}
this.decoderReadPaused = true
try {
this.transport.pauseReads?.()
} catch (error) {
this.handleProtocolError(error)
}
}
private resumeDecoderReads(): void {
if (!this.decoderReadPaused) {
return
}
this.decoderReadPaused = false
if (this.disposed) {
return
}
this.rebaseHealthClocks(Date.now())
try {
this.transport.resumeReads?.()
} catch (error) {
this.handleProtocolError(error)
}
}
private handleWriterSaturationChange(saturated: boolean): void {
this.writerSaturated = saturated
if (!saturated && !this.disposed) {
this.rebaseHealthClocks(Date.now())
}
}
private rebaseHealthClocks(now: number): void {
this.lastReceivedAt = now
for (const seq of this.unackedTimestamps.keys()) {
this.unackedTimestamps.set(seq, now)
}
}
}
function messageLane(msg: JsonRpcMessage): MultiplexerWriterLane {
return 'method' in msg && msg.method === 'pty.data' ? 'ordinary' : 'control'
}
@@ -0,0 +1,284 @@
import { EventEmitter } from 'node:events'
import { describe, expect, it, vi } from 'vitest'
import {
MULTIPLEXER_CONTROL_RESERVE_BYTES,
MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES,
SshMultiplexerTransportWriter,
type MultiplexerTransport,
type MultiplexerWriteSettlement
} from './ssh-multiplexer-transport-writer'
type WriterHarness = {
transport: MultiplexerTransport
drain: () => void
writes: Buffer[]
callbacks: ((result: MultiplexerWriteSettlement) => void)[]
removeDrain: ReturnType<typeof vi.fn>
}
function transportHarness(writeResults: (boolean | void)[]): WriterHarness {
const emitter = new EventEmitter()
const writes: Buffer[] = []
const callbacks: ((result: MultiplexerWriteSettlement) => void)[] = []
const removeDrain = vi.fn()
return {
transport: {
write: (data, onSettled) => {
writes.push(data)
callbacks.push(onSettled!)
return writeResults.shift()
},
supportsWriteSettlement: true,
onDrain: (callback) => {
emitter.on('drain', callback)
return () => {
removeDrain()
emitter.off('drain', callback)
}
},
onData: vi.fn(),
onClose: vi.fn()
},
drain: () => emitter.emit('drain'),
writes,
callbacks,
removeDrain
}
}
describe('SshMultiplexerTransportWriter', () => {
it('selects queued control before ordinary backlog at the drain boundary', () => {
const harness = transportHarness([false, true, true, true])
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary')
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary')
writer.enqueue(Buffer.from('ordinary-3'), 'ordinary')
writer.enqueue(Buffer.from('control'), 'control')
expect(harness.writes.map(String)).toEqual(['ordinary-1'])
harness.drain()
expect(harness.writes.map(String)).toEqual([
'ordinary-1',
'control',
'ordinary-2',
'ordinary-3'
])
writer.dispose()
})
it('preserves FIFO within each lane and prevents ordinary starvation', () => {
const harness = transportHarness([false, ...Array<boolean>(8).fill(true)])
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary')
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary')
for (let index = 1; index <= 6; index++) {
writer.enqueue(Buffer.from(`control-${index}`), 'control')
}
harness.drain()
expect(harness.writes.map(String)).toEqual([
'ordinary-1',
'control-1',
'control-2',
'control-3',
'control-4',
'ordinary-2',
'control-5',
'control-6'
])
writer.dispose()
})
it('waits for drain and settles each write once', () => {
const harness = transportHarness([false, true, true])
const failed = vi.fn()
const writer = new SshMultiplexerTransportWriter(harness.transport, failed)
const settlements = [vi.fn(), vi.fn(), vi.fn()]
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary', settlements[0])
writer.enqueue(Buffer.from('control'), 'control', settlements[1])
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary', settlements[2])
expect(harness.writes.map(String)).toEqual(['ordinary-1'])
harness.callbacks[0]({ ok: true })
expect(settlements[0]).toHaveBeenCalledWith({ ok: true })
expect(harness.writes.map(String)).toEqual(['ordinary-1'])
harness.drain()
expect(harness.writes.map(String)).toEqual(['ordinary-1', 'control', 'ordinary-2'])
harness.callbacks[1]({ ok: true })
harness.callbacks[2]({ ok: true })
expect(settlements.every((settle) => settle.mock.calls.length === 1)).toBe(true)
expect(failed).not.toHaveBeenCalled()
})
it('allows one coalesced liveness bypass per saturated write settlement', () => {
const harness = transportHarness([false, true, true, true])
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary')
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary')
expect(writer.enqueue(Buffer.from('liveness-1'), 'liveness')).toBe(true)
expect(writer.enqueue(Buffer.from('liveness-coalesced'), 'liveness')).toBe(false)
expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1'])
harness.callbacks[1]({ ok: true })
expect(writer.enqueue(Buffer.from('liveness-2'), 'liveness')).toBe(true)
expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1', 'liveness-2'])
harness.drain()
expect(harness.writes.map(String)).toEqual([
'ordinary-1',
'liveness-1',
'liveness-2',
'ordinary-2'
])
writer.dispose()
})
it('reports saturated epochs and their drain boundary exactly once', () => {
const harness = transportHarness([false])
const saturation = vi.fn()
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn(), saturation)
writer.enqueue(Buffer.from('ordinary'), 'ordinary')
writer.enqueue(Buffer.from('queued'), 'ordinary')
expect(saturation.mock.calls).toEqual([[true]])
harness.drain()
expect(saturation.mock.calls).toEqual([[true], [false]])
writer.dispose()
})
it('keeps a full ordinary lane from consuming the control reserve', () => {
const harness = transportHarness([false, true, true])
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
expect(writer.enqueue(Buffer.alloc(1), 'ordinary')).toBe(true)
expect(writer.enqueue(Buffer.alloc(MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES - 1), 'ordinary')).toBe(
true
)
expect(writer.enqueue(Buffer.alloc(MULTIPLEXER_CONTROL_RESERVE_BYTES), 'control')).toBe(true)
expect(harness.writes).toHaveLength(1)
harness.drain()
expect(harness.writes).toHaveLength(3)
writer.dispose()
})
it('fails all retained writes once when a callback reports an error', () => {
const harness = transportHarness([false])
const failed = vi.fn()
const writer = new SshMultiplexerTransportWriter(harness.transport, failed)
const first = vi.fn()
const queued = vi.fn()
const error = new Error('broken pipe')
writer.enqueue(Buffer.from('first'), 'ordinary', first)
writer.enqueue(Buffer.from('queued'), 'control', queued)
harness.callbacks[0]({ ok: false, error })
harness.callbacks[0]({ ok: true })
expect(first).toHaveBeenCalledOnce()
expect(first).toHaveBeenCalledWith({ ok: false, error })
expect(queued).toHaveBeenCalledWith({ ok: false, error })
expect(failed).toHaveBeenCalledWith(error)
expect(harness.removeDrain).toHaveBeenCalledOnce()
})
it('closes on queue overflow and rejects the overflowing write', () => {
const harness = transportHarness([false])
const failed = vi.fn()
const writer = new SshMultiplexerTransportWriter(harness.transport, failed)
const retained = vi.fn()
const overflow = vi.fn()
writer.enqueue(Buffer.alloc(MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES), 'ordinary', retained)
expect(writer.enqueue(Buffer.alloc(1), 'ordinary', overflow)).toBe(false)
expect(overflow).toHaveBeenCalledWith({
ok: false,
error: expect.objectContaining({ message: expect.stringContaining('bounded capacity') })
})
expect(retained).toHaveBeenCalledWith({
ok: false,
error: expect.objectContaining({ message: expect.stringContaining('bounded capacity') })
})
expect(failed).toHaveBeenCalledOnce()
})
it('settles legacy callback-less transports on acceptance and drain', () => {
const emitter = new EventEmitter()
const write = vi.fn().mockReturnValueOnce(false).mockReturnValueOnce(undefined)
const transport: MultiplexerTransport = {
write,
onDrain: (callback) => {
emitter.on('drain', callback)
},
onData: vi.fn(),
onClose: vi.fn()
}
const writer = new SshMultiplexerTransportWriter(transport, vi.fn())
const first = vi.fn()
const second = vi.fn()
writer.enqueue(Buffer.from('first'), 'ordinary', first)
writer.enqueue(Buffer.from('second'), 'control', second)
expect(first).not.toHaveBeenCalled()
expect(second).not.toHaveBeenCalled()
emitter.emit('drain')
expect(first).toHaveBeenCalledWith({ ok: true })
expect(second).toHaveBeenCalledWith({ ok: true })
})
it('does not miss a drain emitted synchronously by a hostile transport', () => {
let drain = (): void => {}
const write = vi.fn(() => {
drain()
return false
})
const writer = new SshMultiplexerTransportWriter(
{
write,
onDrain: (callback) => {
drain = callback
},
onData: vi.fn(),
onClose: vi.fn()
},
vi.fn()
)
const first = vi.fn()
const second = vi.fn()
writer.enqueue(Buffer.from('first'), 'ordinary', first)
writer.enqueue(Buffer.from('second'), 'control', second)
expect(write).toHaveBeenCalledTimes(2)
expect(first).toHaveBeenCalledWith({ ok: true })
expect(second).toHaveBeenCalledWith({ ok: true })
})
it('fails deterministically when write(false) has no drain source', () => {
const failed = vi.fn()
const writer = new SshMultiplexerTransportWriter(
{
write: () => false,
onData: vi.fn(),
onClose: vi.fn()
},
failed
)
const settled = vi.fn()
expect(writer.enqueue(Buffer.from('data'), 'ordinary', settled)).toBe(true)
expect(settled).toHaveBeenCalledWith({
ok: false,
error: expect.objectContaining({ message: expect.stringContaining('without drain support') })
})
expect(failed).toHaveBeenCalledOnce()
})
})
@@ -0,0 +1,270 @@
import { HEADER_LENGTH, MAX_MESSAGE_SIZE } from './relay-protocol'
import { SshMultiplexerWriterLaneScheduler } from './ssh-multiplexer-writer-lane-scheduler'
export type MultiplexerWriteSettlement = { ok: true } | { ok: false; error: Error }
export type MultiplexerTransport = {
write: (data: Buffer, onSettled?: (result: MultiplexerWriteSettlement) => void) => boolean | void
onData: (cb: (data: Buffer) => void) => void
onClose: (cb: () => void) => void
onDrain?: (cb: () => void) => void | (() => void)
supportsWriteSettlement?: boolean
pauseReads?: () => void
resumeReads?: () => void
close?: () => void
}
export type MultiplexerWriterLane = 'ordinary' | 'control' | 'liveness'
type WriterEntry = {
data: Buffer
lane: MultiplexerWriterLane
onSettled: (result: MultiplexerWriteSettlement) => void
settled: boolean
}
export const MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES = 2 * 1024 * 1024
export const MULTIPLEXER_CONTROL_RESERVE_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH
const ORDINARY_QUEUE_MAX_FRAMES = 2048
const CONTROL_QUEUE_MAX_FRAMES = 512
function onceSettlement(
callback: (result: MultiplexerWriteSettlement) => void
): (result: MultiplexerWriteSettlement) => void {
let settled = false
return (result) => {
if (settled) {
return
}
settled = true
callback(result)
}
}
export class SshMultiplexerTransportWriter {
private readonly scheduler = new SshMultiplexerWriterLaneScheduler<WriterEntry>()
private readonly inFlight = new Set<WriterEntry>()
private readonly settleOnDrain = new Set<WriterEntry>()
private ordinaryBytes = 0
private controlBytes = 0
private ordinaryFrames = 0
private controlFrames = 0
private saturated = false
private writing = false
private drainObservedDuringWrite = false
private pumping = false
private closed = false
private livenessOutstanding = false
private removeDrainListener: (() => void) | null = null
constructor(
private readonly transport: MultiplexerTransport,
private readonly onFailure: (error: Error) => void,
private readonly onSaturationChange: (saturated: boolean) => void = () => {}
) {
if (transport.onDrain) {
const remove = transport.onDrain(() => this.handleDrain())
this.removeDrainListener = typeof remove === 'function' ? remove : null
}
}
enqueue(
data: Buffer,
lane: MultiplexerWriterLane,
onSettled: (result: MultiplexerWriteSettlement) => void = () => {}
): boolean {
const settle = onceSettlement(onSettled)
if (this.closed) {
settle({ ok: false, error: new Error('Multiplexer writer is closed') })
return false
}
if (lane === 'liveness' && this.livenessOutstanding) {
return false
}
const admissionError = this.admissionError(data.length, lane)
if (admissionError) {
settle({ ok: false, error: admissionError })
this.fail(admissionError)
return false
}
const entry = { data, lane, onSettled: settle, settled: false }
this.retain(entry)
if (lane === 'liveness' && this.saturated) {
this.writeEntry(entry)
} else {
this.scheduler.enqueue(entry, lane)
this.pump()
}
return true
}
dispose(error = new Error('Multiplexer writer disposed')): void {
if (this.closed) {
return
}
this.closed = true
this.saturated = false
this.removeDrainListener?.()
this.removeDrainListener = null
for (const entry of this.scheduler.clear()) {
this.release(entry, { ok: false, error })
}
for (const entry of Array.from(this.inFlight)) {
this.release(entry, { ok: false, error })
}
this.settleOnDrain.clear()
}
private admissionError(bytes: number, lane: MultiplexerWriterLane): Error | null {
const byteLimit =
lane === 'ordinary' ? MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES : MULTIPLEXER_CONTROL_RESERVE_BYTES
const retainedBytes = lane === 'ordinary' ? this.ordinaryBytes : this.controlBytes
const frameLimit = lane === 'ordinary' ? ORDINARY_QUEUE_MAX_FRAMES : CONTROL_QUEUE_MAX_FRAMES
const retainedFrames = lane === 'ordinary' ? this.ordinaryFrames : this.controlFrames
if (retainedBytes + bytes <= byteLimit && retainedFrames < frameLimit) {
return null
}
return new Error(`Multiplexer ${lane} write queue exceeded its bounded capacity`)
}
private pump(): void {
if (this.pumping || this.closed || this.saturated) {
return
}
this.pumping = true
try {
while (!this.closed && !this.saturated) {
const entry = this.scheduler.select()
if (!entry) {
return
}
this.writeEntry(entry)
}
} finally {
this.pumping = false
}
}
private writeEntry(entry: WriterEntry): void {
this.inFlight.add(entry)
let callbackResult: MultiplexerWriteSettlement | undefined
let writeReturned = false
const onWriteSettled = (result: MultiplexerWriteSettlement): void => {
if (!writeReturned) {
callbackResult = result
return
}
this.handleWriteSettlement(entry, result)
}
try {
this.writing = true
this.drainObservedDuringWrite = false
const accepted = this.transport.write(entry.data, onWriteSettled)
this.writing = false
writeReturned = true
if (accepted === false) {
if (!this.transport.onDrain) {
throw new Error('Multiplexer transport returned write(false) without drain support')
}
this.setSaturated(!this.drainObservedDuringWrite)
if (this.transport.supportsWriteSettlement !== true && this.saturated) {
this.settleOnDrain.add(entry)
} else if (this.transport.supportsWriteSettlement !== true) {
this.handleWriteSettlement(entry, { ok: true })
}
} else if (this.transport.supportsWriteSettlement !== true) {
this.handleWriteSettlement(entry, { ok: true })
}
if (callbackResult) {
this.handleWriteSettlement(entry, callbackResult)
}
} catch (error) {
this.writing = false
writeReturned = true
if (callbackResult) {
this.handleWriteSettlement(entry, callbackResult)
}
this.fail(error instanceof Error ? error : new Error(String(error)))
}
}
private handleWriteSettlement(entry: WriterEntry, result: MultiplexerWriteSettlement): void {
if (entry.settled) {
return
}
this.release(entry, result)
if (!result.ok) {
this.fail(result.error)
return
}
this.pump()
}
private handleDrain(): void {
if (this.closed) {
return
}
if (this.writing) {
this.drainObservedDuringWrite = true
return
}
if (!this.saturated) {
return
}
this.setSaturated(false)
for (const entry of Array.from(this.settleOnDrain)) {
this.release(entry, { ok: true })
}
this.settleOnDrain.clear()
this.pump()
}
private retain(entry: WriterEntry): void {
if (entry.lane === 'ordinary') {
this.ordinaryBytes += entry.data.length
this.ordinaryFrames++
} else {
this.controlBytes += entry.data.length
this.controlFrames++
}
if (entry.lane === 'liveness') {
this.livenessOutstanding = true
}
}
private release(entry: WriterEntry, result: MultiplexerWriteSettlement): void {
if (entry.settled) {
return
}
entry.settled = true
this.inFlight.delete(entry)
this.settleOnDrain.delete(entry)
if (entry.lane === 'ordinary') {
this.ordinaryBytes -= entry.data.length
this.ordinaryFrames--
} else {
this.controlBytes -= entry.data.length
this.controlFrames--
}
if (entry.lane === 'liveness') {
this.livenessOutstanding = false
}
entry.onSettled(result)
}
private setSaturated(saturated: boolean): void {
if (this.saturated === saturated) {
return
}
this.saturated = saturated
this.onSaturationChange(saturated)
}
private fail(error: Error): void {
if (this.closed) {
return
}
this.dispose(error)
this.onFailure(error)
}
}
@@ -0,0 +1,73 @@
import type { MultiplexerWriterLane } from './ssh-multiplexer-transport-writer'
const CONTROL_WRITES_BEFORE_ORDINARY = 4
type LaneQueue<T> = {
entries: T[]
head: number
}
function createLaneQueue<T>(): LaneQueue<T> {
return { entries: [], head: 0 }
}
function hasEntries<T>(queue: LaneQueue<T>): boolean {
return queue.head < queue.entries.length
}
function shift<T>(queue: LaneQueue<T>): T | undefined {
const entry = queue.entries[queue.head]
if (entry === undefined) {
return undefined
}
queue.head += 1
if (queue.head === queue.entries.length) {
queue.entries.length = 0
queue.head = 0
}
return entry
}
function clear<T>(queue: LaneQueue<T>): T[] {
const entries = queue.entries.slice(queue.head)
queue.entries.length = 0
queue.head = 0
return entries
}
export class SshMultiplexerWriterLaneScheduler<T extends object> {
private readonly ordinary = createLaneQueue<T>()
private readonly control = createLaneQueue<T>()
private readonly liveness = createLaneQueue<T>()
private controlWritesSinceOrdinary = 0
enqueue(entry: T, lane: MultiplexerWriterLane): void {
this[lane].entries.push(entry)
}
select(): T | undefined {
const liveness = shift(this.liveness)
if (liveness) {
return liveness
}
if (
hasEntries(this.control) &&
(!hasEntries(this.ordinary) ||
this.controlWritesSinceOrdinary < CONTROL_WRITES_BEFORE_ORDINARY)
) {
this.controlWritesSinceOrdinary += 1
return shift(this.control)
}
const ordinary = shift(this.ordinary)
if (ordinary) {
this.controlWritesSinceOrdinary = 0
return ordinary
}
return shift(this.control)
}
clear(): T[] {
this.controlWritesSinceOrdinary = 0
return [...clear(this.liveness), ...clear(this.control), ...clear(this.ordinary)]
}
}
@@ -0,0 +1,140 @@
import { describe, expect, it, vi } from 'vitest'
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
import { openSshPtyConsumerSession } from './ssh-pty-consumer-session'
function muxReturning(result: unknown): {
mux: SshChannelMultiplexer
request: ReturnType<typeof vi.fn>
} {
const request = vi.fn().mockResolvedValue(result)
return { mux: { request } as unknown as SshChannelMultiplexer, request }
}
function legacyOwnerGrant(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
protocolVersion: 1,
serverBuildId: 'build-a',
clientGeneration: 3,
role: 'session-owner',
ownerGeneration: 7,
ownerLease: 'lease-a',
...overrides
}
}
describe('openSshPtyConsumerSession', () => {
it('makes openClient the one request needed for token-free legacy readiness', async () => {
const { mux, request } = muxReturning(legacyOwnerGrant())
await expect(
openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a'
})
).resolves.toEqual({
mode: 'negotiated',
clientInstanceId: 'client-a',
clientGeneration: 3,
ownerGeneration: 7,
ownerLease: 'lease-a'
})
expect(request).toHaveBeenCalledWith(
'pty.openClient',
{
protocolVersion: 1,
clientInstanceId: 'client-a',
requestedRole: 'session-owner'
},
{ timeoutMs: 10_000 }
)
})
it('carries recovery generation and lease on reconnect', async () => {
const { mux, request } = muxReturning(
legacyOwnerGrant({ ownerGeneration: 8, ownerLease: 'lease-b' })
)
await openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a',
resume: { ownerGeneration: 7, ownerLease: 'lease-a' }
})
expect(request.mock.calls[0][1]).toMatchObject({
resume: { ownerGeneration: 7, ownerLease: 'lease-a' }
})
})
it('rejects a prior or mismatched relay build', async () => {
const { mux } = muxReturning(legacyOwnerGrant({ serverBuildId: 'old-build' }))
await expect(
openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a'
})
).rejects.toThrow('session contract mismatch')
})
it('does not silently downgrade when V1 was offered', async () => {
const { mux } = muxReturning(legacyOwnerGrant())
await expect(
openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a',
outputFlowControl: { requestedWindowSu: 64 }
})
).rejects.toThrow('did not grant')
})
it('rejects an unoffered V1 capability in a legacy session', async () => {
const { mux } = muxReturning(
legacyOwnerGrant({
capabilities: { outputFlowControl: { version: 1, windowSu: 64 } }
})
)
await expect(
openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a'
})
).rejects.toThrow('unoffered')
})
it('uses explicit token-free fallback only for same-build method-not-found', async () => {
const error = Object.assign(new Error('Method not found: pty.openClient'), { code: -32601 })
const request = vi.fn().mockRejectedValue(error)
const mux = { request } as unknown as SshChannelMultiplexer
await expect(
openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a',
allowSameBuildLegacyFallback: true,
outputFlowControl: { requestedWindowSu: 64 }
})
).resolves.toEqual({
mode: 'legacy-fallback',
clientInstanceId: 'client-a',
serverBuildId: 'build-a'
})
})
it.each([
Object.assign(new Error('timeout'), { code: 'TIMEOUT' }),
Object.assign(new Error('auth failed'), { code: -32000 }),
Object.assign(new Error('method missing'), { code: -32601 })
])('does not downgrade an unproved or non-method-not-found error', async (error) => {
const request = vi.fn().mockRejectedValue(error)
const mux = { request } as unknown as SshChannelMultiplexer
await expect(
openSshPtyConsumerSession(mux, {
clientInstanceId: 'client-a',
expectedServerBuildId: 'build-a',
allowSameBuildLegacyFallback: error.code !== -32601
})
).rejects.toBe(error)
})
})
+143
View File
@@ -0,0 +1,143 @@
import {
PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
type PtyConsumerSessionGrant
} from '../../shared/pty-consumer-session'
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
export const SSH_PTY_OPEN_CLIENT_METHOD = 'pty.openClient'
export const SSH_PTY_OPEN_CLIENT_TIMEOUT_MS = 10_000
export type SshPtyConsumerOwnerState = {
mode: 'negotiated'
clientInstanceId: string
clientGeneration: number
ownerGeneration: number
ownerLease: string
outputFlowControl?: {
version: 1
windowSu: number
}
}
export type SshPtyLegacyFallbackState = {
mode: 'legacy-fallback'
clientInstanceId: string
serverBuildId: string
}
export type SshPtyConsumerSessionState = SshPtyConsumerOwnerState | SshPtyLegacyFallbackState
export type OpenSshPtyConsumerSessionOptions = {
clientInstanceId: string
expectedServerBuildId: string | undefined
resume?: Pick<SshPtyConsumerOwnerState, 'ownerGeneration' | 'ownerLease'>
outputFlowControl?: {
requestedWindowSu: number
}
allowSameBuildLegacyFallback?: boolean
}
function validateGrant(
value: unknown,
options: OpenSshPtyConsumerSessionOptions
): PtyConsumerSessionGrant {
if (typeof value !== 'object' || value === null) {
throw new Error('Remote relay returned an invalid pty.openClient grant')
}
if (!options.expectedServerBuildId) {
throw new Error('Local relay build identity is unavailable')
}
const grant = value as Partial<PtyConsumerSessionGrant>
if (
grant.protocolVersion !== PTY_CONSUMER_SESSION_PROTOCOL_VERSION ||
grant.serverBuildId !== options.expectedServerBuildId
) {
throw new Error(
`Remote relay session contract mismatch — expected build ${options.expectedServerBuildId}, got ${grant.serverBuildId ?? 'unknown'}`
)
}
if (
!Number.isSafeInteger(grant.clientGeneration) ||
grant.clientGeneration! <= 0 ||
grant.role !== 'session-owner' ||
!Number.isSafeInteger(grant.ownerGeneration) ||
grant.ownerGeneration! <= 0 ||
typeof grant.ownerLease !== 'string' ||
grant.ownerLease.length === 0
) {
throw new Error('Remote relay did not grant an authenticated PTY session owner')
}
const requestedFlow = options.outputFlowControl
const grantedFlow = grant.capabilities?.outputFlowControl
if (requestedFlow) {
if (
grantedFlow?.version !== 1 ||
!Number.isSafeInteger(grantedFlow.windowSu) ||
grantedFlow.windowSu <= 0 ||
grantedFlow.windowSu > requestedFlow.requestedWindowSu
) {
throw new Error('Remote relay did not grant the offered PTY output-flow-control capability')
}
} else if (grantedFlow) {
throw new Error('Remote relay granted an unoffered PTY output-flow-control capability')
}
return grant as PtyConsumerSessionGrant
}
export async function openSshPtyConsumerSession(
mux: SshChannelMultiplexer,
options: OpenSshPtyConsumerSessionOptions
): Promise<SshPtyConsumerSessionState> {
let result: unknown
try {
result = await mux.request(
SSH_PTY_OPEN_CLIENT_METHOD,
{
protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
clientInstanceId: options.clientInstanceId,
requestedRole: 'session-owner',
...(options.resume ? { resume: options.resume } : {}),
...(options.outputFlowControl
? {
capabilities: {
outputFlowControl: {
versions: [1],
requestedWindowSu: options.outputFlowControl.requestedWindowSu
}
}
}
: {})
},
{ timeoutMs: SSH_PTY_OPEN_CLIENT_TIMEOUT_MS }
)
} catch (error) {
const code = (error as { code?: unknown })?.code
if (
code === -32601 &&
options.allowSameBuildLegacyFallback === true &&
typeof options.expectedServerBuildId === 'string' &&
options.expectedServerBuildId.length > 0
) {
return Object.freeze({
mode: 'legacy-fallback',
clientInstanceId: options.clientInstanceId,
serverBuildId: options.expectedServerBuildId
})
}
throw error
}
const grant = validateGrant(result, options)
return {
mode: 'negotiated',
clientInstanceId: options.clientInstanceId,
clientGeneration: grant.clientGeneration,
ownerGeneration: grant.ownerGeneration!,
ownerLease: grant.ownerLease!,
...(grant.capabilities?.outputFlowControl
? { outputFlowControl: grant.capabilities.outputFlowControl }
: {})
}
}
export const SSH_PTY_SOURCE_WINDOW_SU = DEFAULT_PTY_SOURCE_WINDOW_SU
@@ -0,0 +1,43 @@
import { describe, expect, it } from 'vitest'
import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget'
describe('SshPtyRecoveryRetentionBudget', () => {
it('bounds fragmented recovery by per-PTY and session source, charged bytes, and frames', () => {
const budget = new SshPtyRecoveryRetentionBudget({
perPtySourceSu: 4,
perPtyBytes: 300,
perPtyFrames: 2,
sessionSourceSu: 6,
sessionBytes: 500,
sessionFrames: 3
})
expect(budget.tryRetain('pty-1', 'aa', 2)).toBe(true)
expect(budget.tryRetain('pty-1', 'bb', 2)).toBe(true)
expect(budget.tryRetain('pty-1', 'c', 0)).toBe(false)
expect(budget.tryRetain('pty-2', 'cc', 2)).toBe(true)
expect(budget.snapshot()).toEqual({ sourceSu: 6, bytes: 396, frames: 3, ptys: 2 })
expect(budget.tryRetain('pty-2', 'd', 1)).toBe(false)
budget.release('pty-1')
expect(budget.snapshot()).toEqual({ sourceSu: 2, bytes: 132, frames: 1, ptys: 1 })
expect(budget.tryRetain('pty-2', 'dd', 2)).toBe(true)
})
it('charges UTF-16 storage plus one record before aggregate admission', () => {
const budget = new SshPtyRecoveryRetentionBudget({
perPtySourceSu: 100,
perPtyBytes: 1_000,
perPtyFrames: 10,
sessionSourceSu: 100,
sessionBytes: 500,
sessionFrames: 10
})
expect(budget.tryRetain('pty-1', '\u0000'.repeat(64), 1)).toBe(true)
expect(budget.snapshot().bytes).toBe(256)
expect(budget.tryRetain('pty-2', '\u0000'.repeat(64), 1)).toBe(false)
expect(budget.snapshot()).toEqual({ sourceSu: 1, bytes: 256, frames: 1, ptys: 1 })
})
})
@@ -0,0 +1,95 @@
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
import { chargedPtyRetainedStringBytes } from '../../shared/pty-retained-string-memory'
export const SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU = DEFAULT_PTY_SOURCE_WINDOW_SU
export const SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES = 2 * 1024 * 1024
export const SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES = 1_024
export const SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU = 50 * DEFAULT_PTY_SOURCE_WINDOW_SU
export const SSH_PTY_RECOVERY_SESSION_MAX_BYTES = 64 * 1024 * 1024
export const SSH_PTY_RECOVERY_SESSION_MAX_FRAMES = 64 * 1_024
export type SshPtyRecoveryRetentionLimits = Readonly<{
perPtySourceSu: number
perPtyBytes: number
perPtyFrames: number
sessionSourceSu: number
sessionBytes: number
sessionFrames: number
}>
type RetainedPty = {
sourceSu: number
bytes: number
frames: number
}
const DEFAULT_LIMITS: SshPtyRecoveryRetentionLimits = Object.freeze({
perPtySourceSu: SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU,
perPtyBytes: SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES,
perPtyFrames: SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES,
sessionSourceSu: SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU,
sessionBytes: SSH_PTY_RECOVERY_SESSION_MAX_BYTES,
sessionFrames: SSH_PTY_RECOVERY_SESSION_MAX_FRAMES
})
export class SshPtyRecoveryRetentionBudget {
private readonly retainedByPty = new Map<string, RetainedPty>()
private sourceSu = 0
private bytes = 0
private frames = 0
constructor(private readonly limits: SshPtyRecoveryRetentionLimits = DEFAULT_LIMITS) {}
tryRetain(ptyId: string, data: string, sourceSu: number): boolean {
if (!Number.isSafeInteger(sourceSu) || sourceSu < 0) {
return false
}
const retained = this.retainedByPty.get(ptyId) ?? { sourceSu: 0, bytes: 0, frames: 0 }
const chargedBytes = chargedPtyRetainedStringBytes(data)
if (
retained.sourceSu + sourceSu > this.limits.perPtySourceSu ||
retained.bytes + chargedBytes > this.limits.perPtyBytes ||
retained.frames + 1 > this.limits.perPtyFrames ||
this.sourceSu + sourceSu > this.limits.sessionSourceSu ||
this.bytes + chargedBytes > this.limits.sessionBytes ||
this.frames + 1 > this.limits.sessionFrames
) {
return false
}
retained.sourceSu += sourceSu
retained.bytes += chargedBytes
retained.frames++
this.retainedByPty.set(ptyId, retained)
this.sourceSu += sourceSu
this.bytes += chargedBytes
this.frames++
return true
}
release(ptyId: string): void {
const retained = this.retainedByPty.get(ptyId)
if (!retained) {
return
}
this.retainedByPty.delete(ptyId)
this.sourceSu -= retained.sourceSu
this.bytes -= retained.bytes
this.frames -= retained.frames
}
clear(): void {
this.retainedByPty.clear()
this.sourceSu = 0
this.bytes = 0
this.frames = 0
}
snapshot(): Readonly<{ sourceSu: number; bytes: number; frames: number; ptys: number }> {
return Object.freeze({
sourceSu: this.sourceSu,
bytes: this.bytes,
frames: this.frames,
ptys: this.retainedByPty.size
})
}
}
@@ -0,0 +1,35 @@
import { describe, expect, it } from 'vitest'
import { SshPtyRetiredSourceDeliveries } from './ssh-pty-retired-source-deliveries'
const source = (deliveryToken: string, relayPtyId = 'pty-1') => ({
relayPtyId,
deliveryToken,
clientGeneration: 2,
ownerGeneration: 3
})
describe('SshPtyRetiredSourceDeliveries', () => {
it('retains only the latest canceled token for each ordered PTY stream', () => {
const retired = new SshPtyRetiredSourceDeliveries()
for (let token = 0; token < 10_000; token++) {
retired.retire(1, source(`token-${token}`))
}
expect(retired.size).toBe(1)
expect(retired.has(1, source('token-9999'))).toBe(true)
expect(retired.has(1, source('token-9998'))).toBe(false)
})
it('retires state at the next activation or PTY exit boundary', () => {
const retired = new SshPtyRetiredSourceDeliveries()
retired.retire(1, source('old-token'))
retired.retire(1, source('other-token', 'pty-2'))
retired.activate('pty-1')
expect(retired.has(1, source('old-token'))).toBe(false)
expect(retired.has(1, source('other-token', 'pty-2'))).toBe(true)
expect(retired.size).toBe(1)
})
})
@@ -0,0 +1,36 @@
import type { SshPtySourceFrame } from '../providers/ssh-pty-source-frame'
type SourceDeliveryIdentity = Pick<
SshPtySourceFrame,
'relayPtyId' | 'deliveryToken' | 'clientGeneration' | 'ownerGeneration'
>
export class SshPtyRetiredSourceDeliveries {
private readonly keyByRelayPtyId = new Map<string, string>()
has(providerGeneration: number, source: SourceDeliveryIdentity): boolean {
return (
this.keyByRelayPtyId.get(source.relayPtyId) === sourceDeliveryKey(providerGeneration, source)
)
}
retire(providerGeneration: number, source: SourceDeliveryIdentity): void {
this.keyByRelayPtyId.set(source.relayPtyId, sourceDeliveryKey(providerGeneration, source))
}
activate(relayPtyId: string): void {
this.keyByRelayPtyId.delete(relayPtyId)
}
clear(): void {
this.keyByRelayPtyId.clear()
}
get size(): number {
return this.keyByRelayPtyId.size
}
}
function sourceDeliveryKey(providerGeneration: number, source: SourceDeliveryIdentity): string {
return `${providerGeneration}\0${source.clientGeneration}\0${source.ownerGeneration}\0${source.deliveryToken}`
}
@@ -104,7 +104,7 @@ describe('cross-version isolation', () => {
//
// We feed enough exec results to walk through the deploy: platform,
// $HOME, isRelayAlreadyInstalled probe, lock acquire, upload (no exec),
// npm install, finalize, socket probe, socket poll, then GC scan.
// npm install, finalize, socket probe, credential publication, socket poll, then GC scan.
const responses: string[] = [
'__ORCA_REMOTE_PLATFORM__ Linux x86_64', // tagged POSIX platform probe
'/home/u', // echo $HOME
@@ -122,6 +122,7 @@ describe('cross-version isolation', () => {
'', // rm -f probe-stderr (best-effort cleanup after probe resolved)
'', // touch .install-complete (finalizeInstall)
'DEAD', // launch socket probe
'', // publish the per-launch credential
'READY', // socket poll
'', // release .install-lock after relay liveness is observable
// GC scan begins here
+36 -3
View File
@@ -9,13 +9,17 @@ import {
RELAY_EXIT_CODE_VERSION_MISMATCH
} from './ssh-relay-version-mismatch-error'
function createMockChannel(): ClientChannel {
type MockChannel = ClientChannel & {
stdin: EventEmitter & { write: ReturnType<typeof vi.fn> }
}
function createMockChannel(): MockChannel {
return Object.assign(new EventEmitter(), {
stderr: Object.assign(new EventEmitter(), { resume: vi.fn() }),
stdin: { write: vi.fn() },
stdin: Object.assign(new EventEmitter(), { write: vi.fn(() => true) }),
close: vi.fn(),
resume: vi.fn()
}) as unknown as ClientChannel
}) as unknown as MockChannel
}
// execCommand only rejects with Error; narrow the caught reason (its resolve
@@ -204,6 +208,35 @@ describe('waitForSentinel', () => {
expect(Buffer.concat(chunks)).toEqual(postSentinelPayload)
expect(channel.close).not.toHaveBeenCalled()
})
it.each(['ssh2 channel', 'system-SSH child stdio'])(
'forwards write(false), callback settlement, and drain for a %s',
async (shape) => {
const channel = createMockChannel()
if (shape.startsWith('system')) {
Object.assign(channel, { _process: new EventEmitter() })
}
const callback = vi.fn()
const drain = vi.fn()
channel.stdin.write.mockImplementation((...args: unknown[]) => {
const onWritten = args.find((arg) => typeof arg === 'function') as (
error?: Error | null
) => void
onWritten(null)
return false
})
const transportPromise = waitForSentinel(channel)
channel.emit('data', Buffer.from(RELAY_SENTINEL))
const transport = await transportPromise
transport.onDrain?.(drain)
expect(transport.write(Buffer.from('frame'), callback)).toBe(false)
expect(callback).toHaveBeenCalledWith({ ok: true })
channel.stdin.emit('drain')
expect(drain).toHaveBeenCalledOnce()
expect(transport.supportsWriteSettlement).toBe(true)
}
)
})
describe('execCommand', () => {

Some files were not shown because too many files have changed in this diff Show More